1use crate::error::FttsError;
27use ftts_core::{
28 CancellationToken, EngineError, FrameGenerator, GenerationError, NormalizationOptions,
29 NormalizationTrace, PreparedText, SynthesisObserver, SynthesisRequest, TextPreparationError,
30 TextPreparer, TtsEngine,
31};
32use ftts_model_qwen::checkpoint::{
33 CODEC_LANGUAGE_ENGLISH_ID, CheckpointError, CodecCheckpoint, TALKER_HIDDEN, TalkerCheckpoint,
34};
35use ftts_model_qwen::generate::{QwenGenerator, QwenGeneratorConfig};
36use ftts_model_qwen::microdecoder::MicrodecoderConfig;
37use ftts_model_qwen::prompt::{CloneMode, PromptMode};
38use ftts_model_qwen::sampler::SamplingMode;
39use ftts_model_qwen::speaker::{
40 Encoder as SpeakerEncoder, SPEAKER_SAMPLE_RATE_HZ, log_mel_from_24khz_pcm,
41};
42use ftts_model_qwen::talker::TalkerConfig;
43use ftts_model_qwen::tokenizer::{QwenTokenizer, TokenizerFiles};
44use std::fs;
45use std::fs::OpenOptions;
46use std::path::{Path, PathBuf};
47use symphonia::core::audio::SampleBuffer;
48use symphonia::core::codecs::DecoderOptions;
49use symphonia::core::errors::Error as SymphoniaError;
50use symphonia::core::formats::FormatOptions;
51use symphonia::core::io::MediaSourceStream;
52use symphonia::core::meta::MetadataOptions;
53use symphonia::core::probe::Hint;
54use symphonia::default::{get_codecs, get_probe};
55
56pub const SPEAKER_VECTOR_BYTES: usize = TALKER_HIDDEN * 4;
58
59const CANONICAL_MODEL_BASENAME: &str = "qwen3-tts-12hz-0.6b-base.fttsq";
60
61fn checkpoint_error(error: CheckpointError) -> FttsError {
62 FttsError::ArtifactFormat(error.to_string())
63}
64
65#[derive(Clone, Debug)]
67pub struct ModelBundle {
68 pub root: PathBuf,
70 pub main: PathBuf,
73 pub canonical_main: Option<PathBuf>,
75 pub codec: PathBuf,
77}
78
79impl ModelBundle {
80 pub fn resolve(model: &Path) -> Result<Self, FttsError> {
88 let root = if model.is_dir() {
89 model.to_path_buf()
90 } else {
91 model
92 .parent()
93 .ok_or_else(|| {
94 FttsError::ModelNotFound(format!(
95 "model path {} has no parent directory",
96 model.display()
97 ))
98 })?
99 .to_path_buf()
100 };
101 let canonical_main = if model.is_dir() {
102 let canonical = root.join(CANONICAL_MODEL_BASENAME);
103 if canonical.is_file() {
104 Some(canonical)
105 } else {
106 None
107 }
108 } else if model.extension().and_then(|extension| extension.to_str()) == Some("fttsq") {
109 Some(model.to_path_buf())
110 } else {
111 None
112 };
113 let main = root.join("model.safetensors");
114 let codec = root.join("speech_tokenizer/model.safetensors");
115 let (main_label, main_path) = match canonical_main.as_ref() {
116 Some(path) => ("canonical talker artifact", path),
117 None => ("talker checkpoint", &main),
118 };
119 for (label, path) in [
120 (main_label, main_path),
121 ("codec checkpoint", &codec),
122 ("tokenizer vocabulary", &root.join("vocab.json")),
123 ("tokenizer merges", &root.join("merges.txt")),
124 ("tokenizer config", &root.join("tokenizer_config.json")),
125 ] {
126 if !path.is_file() {
127 return Err(FttsError::ModelNotFound(format!(
128 "{label} is missing at {}; `ftts say` needs a complete model directory \
129 ({CANONICAL_MODEL_BASENAME} or model.safetensors, \
130 speech_tokenizer/model.safetensors, \
131 vocab.json, merges.txt, tokenizer_config.json)",
132 path.display()
133 )));
134 }
135 }
136 Ok(Self {
137 root,
138 main,
139 canonical_main,
140 codec,
141 })
142 }
143}
144
145pub struct LoadedModel {
147 talker: TalkerCheckpoint,
148 codec: CodecCheckpoint,
149 tokenizer: QwenTokenizer,
150 artifact: Option<std::sync::Arc<ftts_artifacts::fttsq::MappedFttsq>>,
155}
156
157impl LoadedModel {
158 pub fn load(bundle: &ModelBundle) -> Result<Self, FttsError> {
164 let read = |name: &str| -> Result<String, FttsError> {
165 let path = bundle.root.join(name);
166 fs::read_to_string(&path).map_err(|error| {
167 FttsError::ArtifactFormat(format!("cannot read {}: {error}", path.display()))
168 })
169 };
170 let vocab = read("vocab.json")?;
171 let merges = read("merges.txt")?;
172 let config = read("tokenizer_config.json")?;
173
174 let (talker, codec, tokenizer) = std::thread::scope(|scope| {
178 let codec = scope.spawn(|| CodecCheckpoint::load(&bundle.codec));
179 let tokenizer = scope.spawn(|| {
180 QwenTokenizer::from_files_using_environment(TokenizerFiles {
181 vocab_json: &vocab,
182 merges_txt: &merges,
183 tokenizer_config_json: &config,
184 })
185 });
186 let talker = match bundle.canonical_main.as_deref() {
187 Some(path) => TalkerCheckpoint::load_fttsq_elided(
191 path,
192 ftts_model_qwen::generate::hot_elision_from_environment(),
193 ),
194 None => TalkerCheckpoint::load(&bundle.main),
195 };
196 (
197 talker,
198 codec.join().expect("codec loader panicked"),
199 tokenizer.join().expect("tokenizer builder panicked"),
200 )
201 });
202 let tokenizer = tokenizer
203 .map_err(|error| FttsError::ArtifactFormat(format!("tokenizer unusable: {error}")))?;
204
205 let talker = talker.map_err(checkpoint_error)?;
206 let artifact = talker.artifact().cloned();
209 Ok(Self {
210 talker,
211 codec: codec.map_err(checkpoint_error)?,
212 tokenizer,
213 artifact,
214 })
215 }
216}
217
218pub fn read_speaker_vector(path: &Path) -> Result<Vec<f32>, FttsError> {
228 let bytes = fs::read(path).map_err(|error| {
229 FttsError::Input(format!(
230 "cannot read speaker vector {}: {error}",
231 path.display()
232 ))
233 })?;
234 if bytes.len() != SPEAKER_VECTOR_BYTES {
235 return Err(FttsError::Input(format!(
236 "speaker vector {} is {} bytes; `ftts say --voice` expects exactly {} \
237 ({TALKER_HIDDEN} little-endian f32)",
238 path.display(),
239 bytes.len(),
240 SPEAKER_VECTOR_BYTES
241 )));
242 }
243 let vector: Vec<f32> = bytes
244 .as_chunks::<4>()
245 .0
246 .iter()
247 .map(|quad| f32::from_le_bytes(*quad))
248 .collect();
249 if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
250 return Err(FttsError::Input(format!(
251 "speaker vector {} holds a non-finite value at index {index}; it would poison every \
252 prefill position it is summed into",
253 path.display()
254 )));
255 }
256 Ok(vector)
257}
258
259#[derive(Clone, Copy, Debug)]
262pub struct DenoiseReport {
263 pub before_dbfs: f32,
265 pub after_dbfs: f32,
267}
268
269#[derive(Default)]
274pub struct ReferenceCleanup<'a> {
275 pub denoise: Option<&'a mut Option<DenoiseReport>>,
277 pub dereverb: Option<&'a mut Option<DereverbReport>>,
279}
280
281pub fn speaker_from_voice(
292 bundle: &ModelBundle,
293 path: &Path,
294 cleanup: ReferenceCleanup<'_>,
295) -> Result<Vec<f32>, FttsError> {
296 let bytes = fs::read(path).map_err(|error| {
297 FttsError::Input(format!(
298 "cannot read voice source {}: {error}",
299 path.display()
300 ))
301 })?;
302 if bytes.len() == SPEAKER_VECTOR_BYTES {
303 return decode_speaker_vector(path, &bytes);
304 }
305 let pcm = decode_reference_audio_any(path)?;
306 let ReferenceCleanup { denoise, dereverb } = cleanup;
307 let pcm = match dereverb {
308 Some(report) => {
309 let before = reverb_time_s(&pcm);
310 let dried = dereverb_reference(&pcm);
311 let after = reverb_time_s(&dried);
312 if let (Some(before), Some(after)) = (before, after) {
313 *report = Some(DereverbReport {
314 before_rt60_s: before,
315 after_rt60_s: after,
316 });
317 }
318 dried
319 }
320 None => pcm,
321 };
322 let pcm = match denoise {
323 Some(report) => {
324 let before = pause_floor_dbfs(&pcm);
325 let cleaned = denoise_reference(&pcm);
326 *report = Some(DenoiseReport {
327 before_dbfs: before,
328 after_dbfs: pause_floor_dbfs(&cleaned),
329 });
330 cleaned
331 }
332 None => pcm,
333 };
334 let mel = log_mel_from_24khz_pcm(&pcm)
335 .map_err(|error| FttsError::Input(format!("cannot extract speaker features: {error}")))?;
336 let encoder = match bundle.canonical_main.as_deref() {
337 Some(artifact) => SpeakerEncoder::load_fttsq(artifact),
338 None => SpeakerEncoder::load(&bundle.main),
339 }
340 .map_err(checkpoint_error)?;
341 let vector = encoder.encode(&mel.values, mel.frames);
342 if vector.iter().all(|value| value.is_finite()) {
343 Ok(vector)
344 } else {
345 Err(FttsError::Input(
346 "speaker encoder produced a non-finite x-vector; refusing to condition synthesis"
347 .to_owned(),
348 ))
349 }
350}
351
352pub fn write_speaker_vector_new(path: &Path, vector: &[f32]) -> Result<(), FttsError> {
354 if vector.len() != TALKER_HIDDEN {
355 return Err(FttsError::Input(format!(
356 "cannot write {}-wide speaker vector; expected {TALKER_HIDDEN}",
357 vector.len()
358 )));
359 }
360 if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
361 return Err(FttsError::Input(format!(
362 "cannot write speaker vector with a non-finite value at index {index}"
363 )));
364 }
365 let mut bytes = Vec::with_capacity(SPEAKER_VECTOR_BYTES);
366 for value in vector {
367 bytes.extend_from_slice(&value.to_le_bytes());
368 }
369 use std::io::Write;
370 let mut file = OpenOptions::new()
371 .write(true)
372 .create_new(true)
373 .open(path)
374 .map_err(|error| {
375 FttsError::Input(format!(
376 "cannot create enrolled voice {} without overwriting an existing file: {error}",
377 path.display()
378 ))
379 })?;
380 file.write_all(&bytes).map_err(|error| {
381 FttsError::Input(format!(
382 "cannot write enrolled voice {}: {error}",
383 path.display()
384 ))
385 })
386}
387
388pub fn replace_speaker_vector(path: &Path, vector: &[f32]) -> Result<PathBuf, FttsError> {
398 let backup = path.with_extension("spk.bak");
399 fs::copy(path, &backup).map_err(|error| {
400 FttsError::Input(format!(
401 "cannot back up the existing voice {} to {}: {error}",
402 path.display(),
403 backup.display()
404 ))
405 })?;
406 let staging = path.with_extension("spk.incoming");
409 if staging.exists() {
410 fs::remove_file(&staging).map_err(|error| {
411 FttsError::Input(format!(
412 "cannot clear the stale staging file {}: {error}",
413 staging.display()
414 ))
415 })?;
416 }
417 write_speaker_vector_new(&staging, vector)?;
418 fs::rename(&staging, path).map_err(|error| {
419 FttsError::Input(format!(
420 "cannot replace {} with the new voice: {error}",
421 path.display()
422 ))
423 })?;
424 Ok(backup)
425}
426
427fn decode_speaker_vector(path: &Path, bytes: &[u8]) -> Result<Vec<f32>, FttsError> {
428 let vector: Vec<f32> = bytes
429 .as_chunks::<4>()
430 .0
431 .iter()
432 .map(|quad| f32::from_le_bytes(*quad))
433 .collect();
434 if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
435 return Err(FttsError::Input(format!(
436 "speaker vector {} holds a non-finite value at index {index}; it would poison every \
437 prefill position it is summed into",
438 path.display()
439 )));
440 }
441 Ok(vector)
442}
443
444const SYSTEM_DECODED_EXTENSIONS: [&str; 6] = ["m4a", "mp3", "aac", "mp4", "ogg", "opus"];
448
449fn decode_reference_audio_any(path: &Path) -> Result<Vec<f32>, FttsError> {
455 let extension = path
456 .extension()
457 .and_then(|extension| extension.to_str())
458 .map(str::to_ascii_lowercase);
459 let needs_system_decoder = extension
460 .as_deref()
461 .is_some_and(|extension| SYSTEM_DECODED_EXTENSIONS.contains(&extension));
462 if !needs_system_decoder {
463 return decode_reference_audio(path);
464 }
465
466 let staging = std::env::temp_dir().join(format!(
467 "ftts-enroll-{}-{}.wav",
468 std::process::id(),
469 path.file_stem()
470 .and_then(|stem| stem.to_str())
471 .unwrap_or("reference")
472 ));
473 let attempts: &[(&str, Vec<&std::ffi::OsStr>)] = &[
474 (
479 "afconvert",
480 vec![
481 "-f".as_ref(),
482 "WAVE".as_ref(),
483 "-d".as_ref(),
484 "LEI16@24000".as_ref(),
485 "-c".as_ref(),
486 "1".as_ref(),
487 path.as_os_str(),
488 staging.as_os_str(),
489 ],
490 ),
491 (
492 "ffmpeg",
493 vec![
494 "-y".as_ref(),
495 "-loglevel".as_ref(),
496 "error".as_ref(),
497 "-i".as_ref(),
498 path.as_os_str(),
499 "-acodec".as_ref(),
500 "pcm_s16le".as_ref(),
501 "-ar".as_ref(),
502 "24000".as_ref(),
503 "-ac".as_ref(),
504 "1".as_ref(),
505 staging.as_os_str(),
506 ],
507 ),
508 ];
509 let mut ran = false;
510 for (tool, arguments) in attempts {
511 match std::process::Command::new(tool).args(arguments).status() {
512 Ok(status) if status.success() => {
513 ran = true;
514 break;
515 }
516 Ok(status) => {
517 let _ = fs::remove_file(&staging);
518 return Err(FttsError::Input(format!(
519 "{tool} failed decoding reference audio {} (exit {status})",
520 path.display()
521 )));
522 }
523 Err(_) => continue, }
525 }
526 if !ran {
527 return Err(FttsError::Input(format!(
528 "reference audio {} is a compressed container and no system decoder was found; \
529 install afconvert (macOS) or ffmpeg, or supply WAV/FLAC",
530 path.display()
531 )));
532 }
533 let decoded = decode_reference_audio(&staging);
534 let _ = fs::remove_file(&staging);
535 decoded
536}
537
538fn decode_reference_audio(path: &Path) -> Result<Vec<f32>, FttsError> {
539 let file = fs::File::open(path).map_err(|error| {
540 FttsError::Input(format!(
541 "cannot open reference audio {}: {error}",
542 path.display()
543 ))
544 })?;
545 let mut hint = Hint::new();
546 if let Some(extension) = path.extension().and_then(|extension| extension.to_str()) {
547 hint.with_extension(extension);
548 }
549 let stream = MediaSourceStream::new(Box::new(file), Default::default());
550 let probed = get_probe()
551 .format(
552 &hint,
553 stream,
554 &FormatOptions::default(),
555 &MetadataOptions::default(),
556 )
557 .map_err(|error| {
558 FttsError::Input(format!(
559 "cannot identify reference audio {}: {error}",
560 path.display()
561 ))
562 })?;
563 let mut format = probed.format;
564 let track = format.default_track().ok_or_else(|| {
565 FttsError::Input(format!(
566 "reference audio {} has no default audio track",
567 path.display()
568 ))
569 })?;
570 let track_id = track.id;
571 let mut decoder = get_codecs()
572 .make(&track.codec_params, &DecoderOptions::default())
573 .map_err(|error| {
574 FttsError::Input(format!(
575 "cannot decode reference audio {}: {error}",
576 path.display()
577 ))
578 })?;
579 let mut sample_rate = None;
580 let mut mono = Vec::new();
581 loop {
582 let packet = match format.next_packet() {
583 Ok(packet) => packet,
584 Err(SymphoniaError::IoError(error))
585 if error.kind() == std::io::ErrorKind::UnexpectedEof =>
586 {
587 break;
588 }
589 Err(error) => {
590 return Err(FttsError::Input(format!(
591 "cannot read reference audio {}: {error}",
592 path.display()
593 )));
594 }
595 };
596 if packet.track_id() != track_id {
597 continue;
598 }
599 let decoded = decoder.decode(&packet).map_err(|error| {
600 FttsError::Input(format!(
601 "cannot decode reference audio {}: {error}",
602 path.display()
603 ))
604 })?;
605 let spec = *decoded.spec();
606 match sample_rate {
607 Some(rate) if rate != spec.rate => {
608 return Err(FttsError::Input(format!(
609 "reference audio {} changed sample rate mid-stream ({rate} to {} Hz)",
610 path.display(),
611 spec.rate
612 )));
613 }
614 None => sample_rate = Some(spec.rate),
615 Some(_) => {}
616 }
617 let channels = spec.channels.count();
618 let mut samples = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
619 samples.copy_interleaved_ref(decoded);
620 for frame in samples.samples().chunks_exact(channels) {
621 mono.push(frame.iter().sum::<f32>() / channels as f32);
622 }
623 }
624 let rate = sample_rate.ok_or_else(|| {
625 FttsError::Input(format!(
626 "reference audio {} contains no decodable samples",
627 path.display()
628 ))
629 })?;
630 if mono.is_empty() {
631 return Err(FttsError::Input(format!(
632 "reference audio {} contains no PCM samples",
633 path.display()
634 )));
635 }
636 let pcm = resample_to_speaker_rate(mono, rate);
637 if pcm.is_empty() {
641 return Err(FttsError::Input(format!(
642 "reference audio {} is too short to resample from {rate} Hz to \
643 {SPEAKER_SAMPLE_RATE_HZ} Hz; supply a longer recording",
644 path.display()
645 )));
646 }
647 Ok(pcm)
648}
649
650fn resample_to_speaker_rate(mono: Vec<f32>, from_rate: u32) -> Vec<f32> {
664 if from_rate == SPEAKER_SAMPLE_RATE_HZ {
665 return mono;
666 }
667 const LOBES: f64 = 6.0;
672 let ratio = f64::from(SPEAKER_SAMPLE_RATE_HZ) / f64::from(from_rate);
673 let cutoff = ratio.min(1.0);
674 let half = (LOBES / cutoff).ceil() as isize;
675 let out_len = ((mono.len() as f64) * ratio).round() as usize;
676
677 let mut out = Vec::with_capacity(out_len);
678 for index in 0..out_len {
679 let center = index as f64 / ratio;
680 let first = center.floor() as isize - half + 1;
681 let mut acc = 0.0_f64;
682 let mut norm = 0.0_f64;
683 for tap in first..first + 2 * half {
684 if tap < 0 {
685 continue;
686 }
687 let Some(sample) = mono.get(tap as usize) else {
688 break;
689 };
690 let weight = lanczos_tap(center - tap as f64, cutoff, LOBES);
691 acc += weight * f64::from(*sample);
692 norm += weight;
693 }
694 out.push(if norm.abs() > 1e-12 {
695 (acc / norm) as f32
696 } else {
697 0.0
698 });
699 }
700 out
701}
702
703const DENOISE_FRAME: usize = 512;
706
707const DENOISE_HOP: usize = DENOISE_FRAME / 4;
710
711const DD_ALPHA: f32 = 0.92;
714
715const OMLSA_GAIN_FLOOR: f32 = 0.017_782_79;
723
724const NOISE_BLOCK_FRAMES: usize = 256;
727
728const SPEECH_ABSENCE_PRIOR: f32 = 0.6;
731
732#[allow(clippy::excessive_precision)]
739const NOISE_QUANTILE_BIAS: f32 = 9.491_221; const NOISE_INIT_QUANTILE: f32 = 0.1;
744
745fn denoise_reference(pcm: &[f32]) -> Vec<f32> {
793 const DENOISE_MIN_FRAMES: usize = 32;
797 if pcm.len() < DENOISE_FRAME + (DENOISE_MIN_FRAMES - 1) * DENOISE_HOP {
798 return pcm.to_vec();
799 }
800 let mut planner = rustfft::FftPlanner::<f32>::new();
801 let forward = planner.plan_fft_forward(DENOISE_FRAME);
802 let inverse = planner.plan_fft_inverse(DENOISE_FRAME);
803
804 let window: Vec<f32> = (0..DENOISE_FRAME)
805 .map(|n| {
806 let phase = std::f32::consts::TAU * n as f32 / DENOISE_FRAME as f32;
807 0.5 - 0.5 * phase.cos()
808 })
809 .collect();
810
811 let bins = DENOISE_FRAME / 2 + 1;
812 let starts: Vec<usize> = (0..=pcm.len() - DENOISE_FRAME)
813 .step_by(DENOISE_HOP)
814 .collect();
815
816 let mut powers: Vec<Vec<f32>> = Vec::with_capacity(starts.len());
819 let mut scratch: Vec<rustfft::num_complex::Complex<f32>> =
820 vec![rustfft::num_complex::Complex::new(0.0, 0.0); DENOISE_FRAME];
821 for &start in &starts {
822 for (slot, n) in scratch.iter_mut().zip(0..DENOISE_FRAME) {
823 *slot = rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0);
824 }
825 forward.process(&mut scratch);
826 powers.push((0..bins).map(|bin| scratch[bin].norm_sqr()).collect());
827 }
828
829 let blocks = powers.len().div_ceil(NOISE_BLOCK_FRAMES);
844 let mut noise = vec![f32::INFINITY; bins];
845 let mut column: Vec<f32> = Vec::with_capacity(NOISE_BLOCK_FRAMES);
846 for block in 0..blocks {
847 let span = block * NOISE_BLOCK_FRAMES..((block + 1) * NOISE_BLOCK_FRAMES).min(powers.len());
848 for (bin, slot) in noise.iter_mut().enumerate() {
849 column.clear();
850 column.extend(powers[span.clone()].iter().map(|frame| frame[bin]));
851 column.sort_by(f32::total_cmp);
852 let rank = ((column.len() as f32 - 1.0) * NOISE_INIT_QUANTILE).round() as usize;
853 *slot = slot.min(column[rank].max(1e-12) * NOISE_QUANTILE_BIAS);
854 }
855 }
856
857 let mut prev_gain = vec![1.0_f32; bins];
859 let mut prev_gamma = vec![1.0_f32; bins];
860
861 let mut out = vec![0.0_f32; pcm.len()];
862 let mut weight = vec![0.0_f32; pcm.len()];
863
864 for (index, &start) in starts.iter().enumerate() {
865 let mut frame: Vec<rustfft::num_complex::Complex<f32>> = (0..DENOISE_FRAME)
866 .map(|n| rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0))
867 .collect();
868 forward.process(&mut frame);
869 let power = &powers[index];
870
871 for bin in 0..bins {
872 let gamma = (power[bin] / noise[bin]).min(1e6);
873 let xi = (DD_ALPHA * prev_gain[bin].powi(2) * prev_gamma[bin]
874 + (1.0 - DD_ALPHA) * (gamma - 1.0).max(0.0))
875 .max(1e-6);
876
877 let nu = (xi / (1.0 + xi)) * gamma;
878 let lsa =
879 ((xi / (1.0 + xi)) * (0.5 * exponential_integral_e1(nu)).exp()).clamp(0.0, 1.0);
880
881 let odds = SPEECH_ABSENCE_PRIOR / (1.0 - SPEECH_ABSENCE_PRIOR);
885 let presence = 1.0 / (1.0 + odds * (1.0 + xi) * (-nu).exp());
886 let presence = presence.clamp(0.0, 1.0);
887
888 let gain = (lsa.max(OMLSA_GAIN_FLOOR).powf(presence)
889 * OMLSA_GAIN_FLOOR.powf(1.0 - presence))
890 .clamp(OMLSA_GAIN_FLOOR, 1.0);
891
892 prev_gain[bin] = gain;
893 prev_gamma[bin] = gamma;
894
895 frame[bin] *= gain;
896 let mirror = DENOISE_FRAME - bin;
897 if mirror != bin && mirror < DENOISE_FRAME {
900 frame[mirror] *= gain;
901 }
902 }
903
904 inverse.process(&mut frame);
905 let scale = 1.0 / DENOISE_FRAME as f32;
906 for n in 0..DENOISE_FRAME {
907 out[start + n] += frame[n].re * scale * window[n];
908 weight[start + n] += window[n] * window[n];
909 }
910 }
911
912 const WOLA_MIN_WEIGHT: f32 = 0.15;
918 for (sample, energy) in out.iter_mut().zip(weight.iter()) {
919 if *energy > WOLA_MIN_WEIGHT {
920 *sample /= *energy;
921 }
922 }
923 let covered =
926 starts.first().copied().unwrap_or(0)..starts.last().map_or(0, |last| last + DENOISE_FRAME);
927 for (index, sample) in out.iter_mut().enumerate() {
928 if !covered.contains(&index) || weight[index] <= WOLA_MIN_WEIGHT {
929 *sample = pcm[index];
930 }
931 }
932 out
933}
934
935fn exponential_integral_e1(x: f32) -> f32 {
942 if x <= 0.0 {
943 return 0.0;
946 }
947 let x = f64::from(x);
948 let value = if x < 1.0 {
949 const A: [f64; 6] = [
951 -0.577_215_664_9,
952 0.999_991_93,
953 -0.249_910_55,
954 0.055_199_68,
955 -0.009_760_04,
956 0.001_078_57,
957 ];
958 let mut acc = 0.0;
959 for (power, coefficient) in A.iter().enumerate() {
960 acc += coefficient * x.powi(power as i32);
961 }
962 acc - x.ln()
963 } else {
964 const A: [f64; 4] = [8.573_328_74, 18.059_016_97, 8.634_760_89, 0.267_773_734];
966 const B: [f64; 4] = [9.573_322_34, 25.632_956_15, 21.099_653_08, 3.958_496_93];
967 let numerator = x.powi(4) + A[0] * x.powi(3) + A[1] * x * x + A[2] * x + A[3];
968 let denominator = x.powi(4) + B[0] * x.powi(3) + B[1] * x * x + B[2] * x + B[3];
969 (numerator / denominator) / (x * x.exp())
970 };
971 value as f32
972}
973
974const DEREVERB_FRAME: usize = 1024;
983const DEREVERB_HOP: usize = 256;
984
985const DEREVERB_TAPS: usize = 40;
987
988const DEREVERB_DELAY: usize = 2;
992
993const DEREVERB_ITERATIONS: usize = 3;
997
998const DEREVERB_LOADING: f64 = 1e-4;
1001
1002#[derive(Clone, Copy, Debug)]
1004pub struct DereverbReport {
1005 pub before_rt60_s: f32,
1007 pub after_rt60_s: f32,
1009}
1010
1011fn dereverb_reference(pcm: &[f32]) -> Vec<f32> {
1043 if pcm.len() < DEREVERB_FRAME * 4 {
1044 return pcm.to_vec();
1045 }
1046 let mut planner = rustfft::FftPlanner::<f32>::new();
1047 let forward = planner.plan_fft_forward(DEREVERB_FRAME);
1048 let inverse = planner.plan_fft_inverse(DEREVERB_FRAME);
1049
1050 let window: Vec<f32> = (0..DEREVERB_FRAME)
1051 .map(|n| {
1052 let phase = std::f32::consts::TAU * n as f32 / DEREVERB_FRAME as f32;
1053 0.5 - 0.5 * phase.cos()
1054 })
1055 .collect();
1056
1057 let bins = DEREVERB_FRAME / 2 + 1;
1058 let starts: Vec<usize> = (0..=pcm.len() - DEREVERB_FRAME)
1059 .step_by(DEREVERB_HOP)
1060 .collect();
1061 let frames = starts.len();
1062 if frames <= DEREVERB_DELAY + DEREVERB_TAPS + 2 {
1063 return pcm.to_vec();
1064 }
1065
1066 let mut observed: Vec<Vec<Complex64>> = Vec::with_capacity(frames);
1068 let mut scratch: Vec<rustfft::num_complex::Complex<f32>> =
1069 vec![rustfft::num_complex::Complex::new(0.0, 0.0); DEREVERB_FRAME];
1070 for &start in &starts {
1071 for (slot, n) in scratch.iter_mut().zip(0..DEREVERB_FRAME) {
1072 *slot = rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0);
1073 }
1074 forward.process(&mut scratch);
1075 observed.push(
1076 scratch[..bins]
1077 .iter()
1078 .map(|value| Complex64::new(f64::from(value.re), f64::from(value.im)))
1079 .collect(),
1080 );
1081 }
1082
1083 let mut desired = observed.clone();
1084 for _ in 0..DEREVERB_ITERATIONS {
1085 for bin in 0..bins {
1086 let mut power: Vec<f64> = (0..frames).map(|t| desired[t][bin].norm_sqr()).collect();
1089 let mean = power.iter().sum::<f64>() / frames as f64;
1090 let floor = (mean * 1e-6).max(1e-12);
1091 for value in &mut power {
1092 *value = value.max(floor);
1093 }
1094
1095 let taps = DEREVERB_TAPS;
1096 let mut covariance = vec![Complex64::new(0.0, 0.0); taps * taps];
1097 let mut cross = vec![Complex64::new(0.0, 0.0); taps];
1098 for t in (DEREVERB_DELAY + taps)..frames {
1099 let weight = 1.0 / power[t];
1100 let regressor: Vec<Complex64> = (0..taps)
1102 .map(|lag| observed[t - DEREVERB_DELAY - lag][bin])
1103 .collect();
1104 for row in 0..taps {
1105 let scaled = regressor[row] * weight;
1106 for column in row..taps {
1107 covariance[row * taps + column] += scaled * regressor[column].conj();
1108 }
1109 cross[row] += scaled * observed[t][bin].conj();
1110 }
1111 }
1112 for row in 0..taps {
1114 for column in 0..row {
1115 covariance[row * taps + column] = covariance[column * taps + row].conj();
1116 }
1117 }
1118 let trace: f64 = (0..taps).map(|i| covariance[i * taps + i].re).sum();
1119 if trace <= 0.0 {
1120 continue;
1121 }
1122 let loading = trace / taps as f64 * DEREVERB_LOADING;
1123 for i in 0..taps {
1124 covariance[i * taps + i] += Complex64::new(loading, 0.0);
1125 }
1126
1127 let Some(filter) = solve_complex_system(&mut covariance, &mut cross, taps) else {
1128 continue;
1129 };
1130 for t in 0..frames {
1131 if t < DEREVERB_DELAY + taps {
1132 desired[t][bin] = observed[t][bin];
1133 continue;
1134 }
1135 let mut tail = Complex64::new(0.0, 0.0);
1136 for (lag, coefficient) in filter.iter().enumerate() {
1137 tail += coefficient.conj() * observed[t - DEREVERB_DELAY - lag][bin];
1138 }
1139 desired[t][bin] = observed[t][bin] - tail;
1140 }
1141 }
1142 }
1143
1144 let mut out = vec![0.0_f32; pcm.len()];
1147 let mut weight = vec![0.0_f32; pcm.len()];
1148 for (index, &start) in starts.iter().enumerate() {
1149 let mut frame = vec![rustfft::num_complex::Complex::new(0.0_f32, 0.0); DEREVERB_FRAME];
1150 for bin in 0..bins {
1151 let value = desired[index][bin];
1152 #[allow(clippy::cast_possible_truncation)]
1153 let value = rustfft::num_complex::Complex::new(value.re as f32, value.im as f32);
1154 frame[bin] = value;
1155 let mirror = DEREVERB_FRAME - bin;
1156 if mirror != bin && mirror < DEREVERB_FRAME {
1157 frame[mirror] = value.conj();
1158 }
1159 }
1160 inverse.process(&mut frame);
1161 let scale = 1.0 / DEREVERB_FRAME as f32;
1162 for n in 0..DEREVERB_FRAME {
1163 out[start + n] += frame[n].re * scale * window[n];
1164 weight[start + n] += window[n] * window[n];
1165 }
1166 }
1167 for (sample, energy) in out.iter_mut().zip(weight.iter()) {
1168 if *energy > 1e-6 {
1169 *sample /= *energy;
1170 }
1171 }
1172 let covered =
1173 starts.first().copied().unwrap_or(0)..starts.last().map_or(0, |last| last + DEREVERB_FRAME);
1174 for (index, sample) in out.iter_mut().enumerate() {
1175 if !covered.contains(&index) || weight[index] <= 1e-6 {
1176 *sample = pcm[index];
1177 }
1178 }
1179 out
1180}
1181
1182type Complex64 = rustfft::num_complex::Complex<f64>;
1187
1188fn solve_complex_system(
1193 a: &mut [Complex64],
1194 b: &mut [Complex64],
1195 n: usize,
1196) -> Option<Vec<Complex64>> {
1197 for column in 0..n {
1198 let (pivot, magnitude) = (column..n).fold((column, 0.0_f64), |best, row| {
1199 let candidate = a[row * n + column].norm_sqr();
1200 if candidate > best.1 {
1201 (row, candidate)
1202 } else {
1203 best
1204 }
1205 });
1206 if magnitude <= f64::MIN_POSITIVE {
1207 return None;
1208 }
1209 if pivot != column {
1210 for k in 0..n {
1211 a.swap(pivot * n + k, column * n + k);
1212 }
1213 b.swap(pivot, column);
1214 }
1215 let diagonal = a[column * n + column];
1216 for row in (column + 1)..n {
1217 let factor = a[row * n + column] / diagonal;
1218 if factor == Complex64::new(0.0, 0.0) {
1219 continue;
1220 }
1221 for k in column..n {
1222 let value = a[column * n + k] * factor;
1223 a[row * n + k] -= value;
1224 }
1225 let value = b[column] * factor;
1226 b[row] -= value;
1227 }
1228 }
1229 let mut solution = vec![Complex64::new(0.0, 0.0); n];
1230 for row in (0..n).rev() {
1231 let mut accumulator = b[row];
1232 for k in (row + 1)..n {
1233 accumulator -= a[row * n + k] * solution[k];
1234 }
1235 solution[row] = accumulator / a[row * n + row];
1236 }
1237 Some(solution)
1238}
1239
1240fn reverb_time_s(pcm: &[f32]) -> Option<f32> {
1247 let hop = (SPEAKER_SAMPLE_RATE_HZ as usize) / 100; if pcm.len() < hop * 32 {
1249 return None;
1250 }
1251 let envelope: Vec<f32> = pcm
1252 .chunks_exact(hop)
1253 .map(|chunk| {
1254 let energy = chunk.iter().map(|s| s * s).sum::<f32>() / chunk.len() as f32;
1255 10.0 * (energy + 1e-9).log10()
1256 })
1257 .collect();
1258 let peak = envelope.iter().copied().fold(f32::NEG_INFINITY, f32::max);
1259 let span = 15_usize; let mut slopes: Vec<f32> = Vec::new();
1261 for index in 1..envelope.len().saturating_sub(span) {
1262 if envelope[index] < peak - 25.0 || envelope[index] <= envelope[index - 1] {
1263 continue;
1264 }
1265 let drop = envelope[index] - envelope[index + span - 1];
1266 if drop < 6.0 {
1267 continue;
1268 }
1269 slopes.push(drop / (span as f32 * 0.01));
1270 }
1271 if slopes.is_empty() {
1272 return None;
1273 }
1274 slopes.sort_by(f32::total_cmp);
1275 let median = slopes[slopes.len() / 2];
1276 (median > 0.0).then(|| 60.0 / median)
1277}
1278
1279fn pause_floor_dbfs(pcm: &[f32]) -> f32 {
1284 let span = (SPEAKER_SAMPLE_RATE_HZ as usize) / 20;
1285 if pcm.len() < span {
1286 return f32::NEG_INFINITY;
1287 }
1288 let mut windows: Vec<f32> = pcm
1289 .chunks_exact(span)
1290 .map(|chunk| {
1291 (chunk
1292 .iter()
1293 .map(|s| f64::from(*s) * f64::from(*s))
1294 .sum::<f64>()
1295 / chunk.len() as f64)
1296 .sqrt() as f32
1297 })
1298 .filter(|rms| *rms > 0.0)
1299 .collect();
1300 if windows.is_empty() {
1301 return f32::NEG_INFINITY;
1302 }
1303 windows.sort_by(f32::total_cmp);
1304 let keep = (windows.len() / 10).max(1);
1305 let mean = windows[..keep].iter().sum::<f32>() / keep as f32;
1306 20.0 * mean.log10()
1307}
1308
1309fn lanczos_tap(offset: f64, cutoff: f64, lobes: f64) -> f64 {
1311 let scaled = cutoff * offset;
1312 if scaled.abs() >= lobes {
1313 return 0.0;
1314 }
1315 sinc(scaled) * sinc(scaled / lobes)
1316}
1317
1318fn sinc(x: f64) -> f64 {
1320 if x.abs() < 1e-12 {
1321 return 1.0;
1322 }
1323 let scaled = std::f64::consts::PI * x;
1324 scaled.sin() / scaled
1325}
1326
1327struct PreparedPassThrough {
1329 prepared: PreparedText,
1330}
1331
1332impl TextPreparer for PreparedPassThrough {
1333 fn prepare(
1334 &self,
1335 _text: &str,
1336 _options: &NormalizationOptions,
1337 ) -> Result<PreparedText, TextPreparationError> {
1338 Ok(PreparedText::new(
1339 self.prepared.token_ids.clone(),
1340 NormalizationTrace {
1341 mode: self.prepared.normalization_trace.mode,
1342 unicode_version: self.prepared.normalization_trace.unicode_version.clone(),
1343 changes: self.prepared.normalization_trace.changes.clone(),
1344 },
1345 ))
1346 }
1347}
1348
1349pub struct SynthesizedAudio {
1351 pub frames: u64,
1353 pub prepared_token_count: usize,
1355 pub pcm: Vec<f32>,
1357 pub ttfa: Option<std::time::Duration>,
1362}
1363
1364#[allow(clippy::too_many_arguments)]
1371pub fn synthesize(
1372 model: &LoadedModel,
1373 engine: &TtsEngine,
1374 request: &SynthesisRequest,
1375 speaker: &[f32],
1376 seed: u64,
1377 cancellation: &CancellationToken,
1378 observer: &dyn SynthesisObserver,
1379) -> Result<SynthesizedAudio, FttsError> {
1380 let prepared_raw = model
1382 .tokenizer
1383 .prepare(&request.text, &request.normalization_options)
1384 .map_err(|error| FttsError::Input(format!("text preparation failed: {error}")))?;
1385 let wrapped = TalkerCheckpoint::wrap_target_ids(&prepared_raw.token_ids);
1386 let prepared = PreparedText::new(wrapped.clone(), prepared_raw.normalization_trace);
1387
1388 let ids = TalkerCheckpoint::utterance_text_ids(&wrapped);
1390 let table = model
1391 .talker
1392 .gather_text_rows(&ids)
1393 .map_err(checkpoint_error)?;
1394
1395 let header = model
1397 .talker
1398 .xvector_header(&table, speaker, CODEC_LANGUAGE_ENGLISH_ID)
1399 .map_err(checkpoint_error)?;
1400 let tts_eos = model.talker.tts_eos(&table);
1401
1402 let talker_layers = model.talker.talker_layer_weights();
1404 let micro_layers = model.talker.microdecoder_layer_weights();
1405 let residual = model.talker.residual_embedding_slices();
1406 let heads = model.talker.microdecoder_head_slices();
1407 let micro_residual = &residual[..residual.len() - 1];
1410
1411 let mut generator = QwenGenerator::new_with_artifact(
1412 QwenGeneratorConfig {
1413 talker_config: TalkerConfig::default(),
1414 talker_weights: model.talker.talker_weights(&talker_layers),
1415 text: model.talker.text_weights(&table),
1416 feedback: model.talker.feedback_tables(&residual),
1417 microdecoder_config: MicrodecoderConfig::default(),
1418 microdecoder_weights: model.talker.microdecoder_weights(
1419 µ_layers,
1420 micro_residual,
1421 &heads,
1422 ),
1423 prompt_mode: PromptMode {
1424 clone_mode: CloneMode::XVector,
1425 non_streaming_mode: false,
1426 },
1427 header,
1428 tts_eos,
1429 reference: None,
1430 sampling_mode: SamplingMode::Production,
1441 seed,
1442 },
1443 model.artifact.as_deref(),
1444 );
1445
1446 let preparer = PreparedPassThrough { prepared };
1455 let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<ftts_core::CodeFrame>(256);
1456 let codec = &model.codec;
1457 let synthesis_started = std::time::Instant::now();
1458 let (result, pcm, ttfa) = std::thread::scope(
1459 |scope| -> Result<
1460 (
1461 ftts_core::SynthesisResult,
1462 Vec<f32>,
1463 Option<std::time::Duration>,
1464 ),
1465 FttsError,
1466 > {
1467 let worker = scope.spawn(
1468 move || -> Result<(Vec<f32>, Option<std::time::Duration>), FttsError> {
1469 ftts_kernels::team::bypass_team_on_this_thread();
1472 const PACKET_FRAMES: usize = 4;
1473 let mut state = codec.stream_state();
1474 let mut pcm = Vec::new();
1475 let mut packet_pcm = Vec::new();
1478 let mut packet: Vec<i32> = Vec::with_capacity(16 * PACKET_FRAMES);
1479 let mut packet_frames = 0_usize;
1480 let mut first_audio_at: Option<std::time::Duration> = None;
1481 while let Ok(frame) = frame_rx.recv() {
1482 if frame.codes.len() != 16 {
1483 return Err(FttsError::Generic(format!(
1484 "generated frame carries {} codes, expected 16",
1485 frame.codes.len()
1486 )));
1487 }
1488 for code in &frame.codes {
1489 packet.push(i32::try_from(*code).map_err(|_| {
1490 FttsError::Generic(format!(
1491 "generated code {code} does not fit the codec's i32"
1492 ))
1493 })?);
1494 }
1495 packet_frames += 1;
1496 if packet_frames == PACKET_FRAMES {
1497 codec
1498 .stream_push(&mut state, &packet, packet_frames, &mut packet_pcm)
1499 .map_err(checkpoint_error)?;
1500 pcm.extend_from_slice(&packet_pcm);
1501 first_audio_at.get_or_insert_with(|| synthesis_started.elapsed());
1502 packet.clear();
1503 packet_frames = 0;
1504 }
1505 }
1506 if packet_frames > 0 {
1507 codec
1508 .stream_push(&mut state, &packet, packet_frames, &mut packet_pcm)
1509 .map_err(checkpoint_error)?;
1510 pcm.extend_from_slice(&packet_pcm);
1511 first_audio_at.get_or_insert_with(|| synthesis_started.elapsed());
1512 }
1513 Ok((pcm, first_audio_at))
1514 },
1515 );
1516
1517 let mut tee = TeeGenerator {
1518 inner: &mut generator,
1519 frames: frame_tx,
1520 };
1521 let result = engine
1522 .synthesize(
1523 request.clone(),
1524 &preparer,
1525 &mut tee as &mut dyn FrameGenerator,
1526 cancellation,
1527 observer,
1528 )
1529 .map_err(engine_error);
1530 drop(tee); let pcm = worker.join().expect("codec worker must not panic");
1532 let result = result?;
1536 let (pcm, ttfa) = pcm?;
1537 Ok((result, pcm, ttfa))
1538 },
1539 )?;
1540
1541 if result.code_frames.is_empty() {
1542 return Err(FttsError::Generic(
1543 "the talker stopped before emitting a frame; there is no audio to write. This is a \
1544 model or prompt problem, not an output problem — check the speaker vector and the \
1545 text"
1546 .to_owned(),
1547 ));
1548 }
1549
1550 Ok(SynthesizedAudio {
1551 frames: result.generated_frames,
1552 prepared_token_count: result.prepared_token_count,
1553 pcm,
1554 ttfa,
1555 })
1556}
1557
1558struct TeeGenerator<'a> {
1564 inner: &'a mut dyn FrameGenerator,
1565 frames: std::sync::mpsc::SyncSender<ftts_core::CodeFrame>,
1566}
1567
1568impl FrameGenerator for TeeGenerator<'_> {
1569 fn begin_utterance(&mut self, prepared: &PreparedText) -> Result<(), GenerationError> {
1570 self.inner.begin_utterance(prepared)
1571 }
1572
1573 fn next_frame(&mut self) -> Result<Option<ftts_core::CodeFrame>, GenerationError> {
1574 let frame = self.inner.next_frame()?;
1575 if let Some(frame) = &frame
1576 && self.frames.send(frame.clone()).is_err()
1577 {
1578 return Err(GenerationError::new(
1579 "the codec worker stopped accepting frames; its error follows at join",
1580 ));
1581 }
1582 Ok(frame)
1583 }
1584}
1585
1586fn engine_error(error: EngineError) -> FttsError {
1588 match error {
1589 EngineError::BudgetExceeded(_) => FttsError::BudgetTimeout(error.to_string()),
1590 EngineError::ResourceAdmission(_) => FttsError::BudgetTimeout(error.to_string()),
1591 EngineError::TextPreparation(_) => FttsError::Input(error.to_string()),
1592 other => FttsError::Generic(other.to_string()),
1593 }
1594}
1595
1596#[must_use]
1598pub fn generation_error(message: &str) -> GenerationError {
1599 GenerationError::new(message)
1600}
1601
1602#[cfg(test)]
1603mod tests {
1604 use super::*;
1605
1606 #[test]
1609 fn audio_at_the_pinned_rate_is_returned_bit_for_bit() {
1610 let pcm: Vec<f32> = (0..4_096)
1611 .map(|n| (n as f32 * 0.017).sin() * 0.4 + (n as f32 * 0.31).sin() * 0.05)
1612 .collect();
1613 let out = resample_to_speaker_rate(pcm.clone(), SPEAKER_SAMPLE_RATE_HZ);
1614 assert_eq!(out.len(), pcm.len());
1615 for (index, (a, b)) in out.iter().zip(pcm.iter()).enumerate() {
1616 assert!(
1617 a.to_bits() == b.to_bits(),
1618 "sample {index} was altered at the pinned rate"
1619 );
1620 }
1621 }
1622
1623 #[test]
1631 fn the_exponential_integral_matches_its_series_expansion() {
1632 for (x, expected) in [
1634 (0.1_f32, 1.822_923_9_f32),
1635 (0.5, 0.559_773_6),
1636 (0.9, 0.260_183_94),
1637 ] {
1638 let actual = exponential_integral_e1(x);
1639 let relative = ((actual - expected) / expected).abs();
1640 assert!(
1641 relative < 1e-6,
1642 "E1({x}) = {actual} but the series gives {expected} (relative {relative:e})"
1643 );
1644 }
1645
1646 let below = exponential_integral_e1(0.999_9);
1648 let above = exponential_integral_e1(1.000_1);
1649 assert!(
1650 below > above && (below - above).abs() < 1e-4,
1651 "the series and rational branches disagree across x = 1: {below} vs {above}"
1652 );
1653 assert_eq!(
1654 exponential_integral_e1(0.0),
1655 0.0,
1656 "a non-positive argument must not produce NaN"
1657 );
1658 }
1659
1660 #[test]
1664 fn denoise_lowers_the_floor_between_bursts_without_eating_the_signal() {
1665 const TONE_HZ: f64 = 700.0;
1666 let samples = SPEAKER_SAMPLE_RATE_HZ as usize * 2;
1667 let mut state = 0x2545_F491_4F6C_DD1D_u64;
1669 let mut noise = || {
1670 state ^= state << 13;
1671 state ^= state >> 7;
1672 state ^= state << 17;
1673 ((state >> 40) as f32 / 16_777_216.0) - 0.5
1674 };
1675
1676 let clean: Vec<f32> = (0..samples)
1678 .map(|n| {
1679 let t = n as f64 / f64::from(SPEAKER_SAMPLE_RATE_HZ);
1680 let speaking = (n / (SPEAKER_SAMPLE_RATE_HZ as usize / 2)).is_multiple_of(2);
1681 if speaking {
1682 (std::f64::consts::TAU * TONE_HZ * t).sin() as f32 * 0.35
1683 } else {
1684 0.0
1685 }
1686 })
1687 .collect();
1688 let noisy: Vec<f32> = clean.iter().map(|s| s + noise() * 0.1).collect();
1692
1693 let cleaned = denoise_reference(&noisy);
1694 assert_eq!(cleaned.len(), noisy.len(), "denoise must preserve length");
1695 assert!(
1696 cleaned.iter().all(|s| s.is_finite()),
1697 "denoise produced a non-finite sample"
1698 );
1699
1700 let before = pause_floor_dbfs(&noisy);
1701 let after = pause_floor_dbfs(&cleaned);
1702 assert!(
1703 after < before - 3.0,
1704 "expected the pause floor to drop by >3 dB, got {before:.1} -> {after:.1} dBFS"
1705 );
1706
1707 let span = SPEAKER_SAMPLE_RATE_HZ as usize / 4;
1709 let peak_rms = |pcm: &[f32]| {
1710 pcm.chunks_exact(span)
1711 .map(|c| (c.iter().map(|s| s * s).sum::<f32>() / c.len() as f32).sqrt())
1712 .fold(0.0_f32, f32::max)
1713 };
1714 let kept = peak_rms(&cleaned) / peak_rms(&noisy);
1715 assert!(
1716 kept > 0.7,
1717 "denoise removed too much of the signal: peak RMS kept {kept:.3} of the original"
1718 );
1719 }
1720
1721 #[test]
1729 fn denoise_keeps_a_reference_that_opens_on_speech() {
1730 let rate = SPEAKER_SAMPLE_RATE_HZ as usize;
1731 let mut state = 0x9E37_79B9_7F4A_7C15_u64;
1732 let mut hiss = || {
1733 state ^= state << 13;
1734 state ^= state >> 7;
1735 state ^= state << 17;
1736 ((state >> 40) as f32 / 16_777_216.0) - 0.5
1737 };
1738 let burst = rate / 4;
1739 let noisy: Vec<f32> = (0..rate * 3)
1740 .map(|n| {
1741 let t = n as f64 / rate as f64;
1742 let index = n / burst;
1743 let voice = if index.is_multiple_of(2) {
1744 let vibrato = 1.0 + 0.03 * (std::f64::consts::TAU * 5.5 * t).sin();
1747 let phase = (n % burst) as f32 / burst as f32;
1748 let envelope = (std::f32::consts::PI * phase).sin();
1749 let f0 = 140.0 * vibrato * (1.0 + 0.15 * (index / 2) as f64);
1750 (1..=10)
1751 .map(|h| {
1752 let a = 0.3 / h as f32;
1753 (std::f64::consts::TAU * f0 * h as f64 * t).sin() as f32 * a
1754 })
1755 .sum::<f32>()
1756 * envelope
1757 } else {
1758 0.0
1759 };
1760 voice + hiss() * 0.02
1761 })
1762 .collect();
1763
1764 let cleaned = denoise_reference(&noisy);
1765 let rms = |pcm: &[f32]| (pcm.iter().map(|s| s * s).sum::<f32>() / pcm.len() as f32).sqrt();
1766 let kept = rms(&cleaned[..burst]) / rms(&noisy[..burst]);
1767 assert!(
1768 kept > 0.7,
1769 "the opening burst kept only {kept:.3} of its energy; the noise floor is being \
1770 seeded from speech the estimator has not yet learned to exclude"
1771 );
1772 }
1773
1774 #[test]
1784 fn denoise_does_not_preferentially_zap_high_frequencies() {
1785 let rate = SPEAKER_SAMPLE_RATE_HZ as usize;
1786 let mut state = 0xDEAD_BEEF_1234_5678_u64;
1787 let mut hiss = || {
1788 state ^= state << 13;
1789 state ^= state >> 7;
1790 state ^= state << 17;
1791 ((state >> 40) as f32 / 16_777_216.0) - 0.5
1792 };
1793 let probes: [f64; 5] = [300.0, 1_200.0, 3_000.0, 6_000.0, 9_000.0];
1796 let burst = rate / 4;
1797 let noisy: Vec<f32> = (0..rate * 3)
1798 .map(|n| {
1799 let t = n as f64 / rate as f64;
1800 let voice = if (n / burst).is_multiple_of(2) {
1801 let phase = (n % burst) as f32 / burst as f32;
1802 let envelope = (std::f32::consts::PI * phase).sin();
1803 probes
1804 .iter()
1805 .map(|hz| (std::f64::consts::TAU * hz * t).sin() as f32 * 0.12)
1806 .sum::<f32>()
1807 * envelope
1808 } else {
1809 0.0
1810 };
1811 voice + hiss() * 0.02
1812 })
1813 .collect();
1814
1815 let cleaned = denoise_reference(&noisy);
1816
1817 let span = burst / 2..burst;
1820 let energy_at = |pcm: &[f32], hz: f64| -> f32 {
1821 let (mut re, mut im) = (0.0_f64, 0.0_f64);
1822 for (offset, sample) in pcm[span.clone()].iter().enumerate() {
1823 let t = (span.start + offset) as f64 / rate as f64;
1824 let angle = std::f64::consts::TAU * hz * t;
1825 re += f64::from(*sample) * angle.cos();
1826 im += f64::from(*sample) * angle.sin();
1827 }
1828 (re.hypot(im) / span.len() as f64) as f32
1829 };
1830
1831 let retention: Vec<f32> = probes
1832 .iter()
1833 .map(|hz| energy_at(&cleaned, *hz) / energy_at(&noisy, *hz).max(1e-9))
1834 .collect();
1835 let low = retention[0];
1836 for (hz, kept) in probes.iter().zip(retention.iter()) {
1837 assert!(
1838 *kept > 0.5,
1839 "{hz} Hz retained only {kept:.3}; the denoiser is eating the band, not the noise \
1840 (all bands: {retention:?})"
1841 );
1842 assert!(
1843 *kept > low * 0.6,
1844 "{hz} Hz retained {kept:.3} against {low:.3} at 300 Hz — high frequencies are \
1845 being attenuated preferentially, which is how sibilance and speaker identity go \
1846 (all bands: {retention:?})"
1847 );
1848 }
1849 }
1850
1851 #[test]
1855 fn a_clip_shorter_than_its_downsample_ratio_resamples_to_nothing() {
1856 let out = resample_to_speaker_rate(vec![0.25], 192_000);
1857 assert!(
1858 out.is_empty(),
1859 "one sample at 192 kHz is less than half an output sample at \
1860 {SPEAKER_SAMPLE_RATE_HZ} Hz, so it cannot produce one"
1861 );
1862 }
1863
1864 #[test]
1868 fn a_48k_tone_resamples_to_24k_with_its_shape_intact() {
1869 const SOURCE_HZ: u32 = 48_000;
1870 const TONE_HZ: f64 = 1_000.0;
1871 let samples = SOURCE_HZ as usize; let pcm: Vec<f32> = (0..samples)
1873 .map(|n| {
1874 let t = n as f64 / f64::from(SOURCE_HZ);
1875 (std::f64::consts::TAU * TONE_HZ * t).sin() as f32
1876 })
1877 .collect();
1878
1879 let out = resample_to_speaker_rate(pcm, SOURCE_HZ);
1880
1881 let expected_len = SPEAKER_SAMPLE_RATE_HZ as usize;
1882 assert!(
1883 out.len().abs_diff(expected_len) <= 1,
1884 "expected ~{expected_len} samples at {SPEAKER_SAMPLE_RATE_HZ} Hz, got {}",
1885 out.len()
1886 );
1887
1888 let skip = 64;
1891 let interior = out.len() - skip;
1892 let mut worst = 0.0_f32;
1893 for (index, sample) in out.iter().enumerate().take(interior).skip(skip) {
1894 let t = index as f64 / f64::from(SPEAKER_SAMPLE_RATE_HZ);
1895 let ideal = (std::f64::consts::TAU * TONE_HZ * t).sin() as f32;
1896 worst = worst.max((sample - ideal).abs());
1897 }
1898 assert!(
1899 worst < 0.02,
1900 "resampled tone drifted from the analytic reference by {worst}"
1901 );
1902 }
1903
1904 #[test]
1905 fn a_short_speaker_vector_is_refused_rather_than_padded() {
1906 let dir = std::env::temp_dir().join("ftts-synth-tests");
1907 fs::create_dir_all(&dir).expect("temp dir");
1908 let path = dir.join("short.spk");
1909 fs::write(&path, vec![0u8; 64]).expect("write");
1910 let error = read_speaker_vector(&path).expect_err("a short vector must be refused");
1911 let message = error.to_string();
1912 assert!(message.contains("64 bytes"), "{message}");
1913 assert!(message.contains("4096"), "{message}");
1914 }
1915
1916 #[test]
1917 fn a_non_finite_speaker_vector_is_refused() {
1918 let dir = std::env::temp_dir().join("ftts-synth-tests");
1919 fs::create_dir_all(&dir).expect("temp dir");
1920 let path = dir.join("nan.spk");
1921 let mut bytes = vec![0u8; SPEAKER_VECTOR_BYTES];
1922 bytes[0..4].copy_from_slice(&f32::NAN.to_le_bytes());
1923 fs::write(&path, &bytes).expect("write");
1924 let error = read_speaker_vector(&path).expect_err("NaN must be refused");
1925 assert!(error.to_string().contains("index 0"), "{error}");
1926 }
1927
1928 #[test]
1929 fn a_well_formed_speaker_vector_reads_back_exactly() {
1930 let dir = std::env::temp_dir().join("ftts-synth-tests");
1931 fs::create_dir_all(&dir).expect("temp dir");
1932 let path = dir.join("good.spk");
1933 let expected: Vec<f32> = (0..TALKER_HIDDEN).map(|i| i as f32 * 0.001).collect();
1934 let mut bytes = Vec::with_capacity(SPEAKER_VECTOR_BYTES);
1935 for value in &expected {
1936 bytes.extend_from_slice(&value.to_le_bytes());
1937 }
1938 fs::write(&path, &bytes).expect("write");
1939 assert_eq!(read_speaker_vector(&path).expect("read"), expected);
1940 }
1941
1942 #[test]
1943 fn enrollment_writer_refuses_overwrite_and_preserves_the_vector() {
1944 let path = std::env::temp_dir().join(format!(
1945 "ftts-enroll-{}-{}.spk",
1946 std::process::id(),
1947 std::time::SystemTime::now()
1948 .duration_since(std::time::UNIX_EPOCH)
1949 .expect("clock")
1950 .as_nanos()
1951 ));
1952 let expected: Vec<f32> = (0..TALKER_HIDDEN)
1953 .map(|index| index as f32 * 0.125)
1954 .collect();
1955 write_speaker_vector_new(&path, &expected).expect("initial enrollment write");
1956 assert_eq!(
1957 read_speaker_vector(&path).expect("read enrolled vector"),
1958 expected
1959 );
1960 let error = write_speaker_vector_new(&path, &[0.0; TALKER_HIDDEN])
1961 .expect_err("an enrollment must never replace an existing voice");
1962 assert!(error.to_string().contains("without overwriting"), "{error}");
1963 }
1964
1965 #[test]
1966 fn wav_reference_decodes_to_mono_24khz_pcm() {
1967 let path = std::env::temp_dir().join(format!(
1968 "ftts-reference-{}-{}.wav",
1969 std::process::id(),
1970 std::time::SystemTime::now()
1971 .duration_since(std::time::UNIX_EPOCH)
1972 .expect("clock")
1973 .as_nanos()
1974 ));
1975 let pcm: Vec<f32> = (0..1_920)
1976 .map(|index| (index as f32 / 1_920.0 * std::f32::consts::TAU).sin() * 0.25)
1977 .collect();
1978 fs::write(
1979 &path,
1980 ftts_core::audio::encode_wav(&pcm, SPEAKER_SAMPLE_RATE_HZ),
1981 )
1982 .expect("write reference WAV");
1983 let decoded = decode_reference_audio(&path).expect("decode reference WAV");
1984 assert_eq!(decoded.len(), pcm.len());
1985 assert!(decoded.iter().all(|sample| sample.is_finite()));
1986 }
1987
1988 #[test]
1989 fn a_bundle_names_the_file_that_is_actually_missing() {
1990 let dir = std::env::temp_dir().join("ftts-bundle-tests-empty");
1993 fs::create_dir_all(&dir).expect("temp dir");
1994 let error = ModelBundle::resolve(&dir).expect_err("an empty directory is not a bundle");
1995 assert!(error.to_string().contains("model.safetensors"), "{error}");
1996 }
1997
1998 #[test]
1999 fn a_complete_bundle_prefers_its_canonical_artifact_for_synthesis() {
2000 let nonce = std::time::SystemTime::now()
2001 .duration_since(std::time::UNIX_EPOCH)
2002 .expect("clock after epoch")
2003 .as_nanos();
2004 let dir = std::env::temp_dir().join(format!(
2005 "ftts-bundle-canonical-{}-{nonce}",
2006 std::process::id()
2007 ));
2008 fs::create_dir_all(dir.join("speech_tokenizer")).expect("create bundle sidecar directory");
2009 for name in [
2010 CANONICAL_MODEL_BASENAME,
2011 "speech_tokenizer/model.safetensors",
2012 "vocab.json",
2013 "merges.txt",
2014 "tokenizer_config.json",
2015 ] {
2016 fs::write(dir.join(name), []).expect("write bundle fixture sidecar");
2017 }
2018
2019 let expected_artifact = dir.join(CANONICAL_MODEL_BASENAME);
2020 let bundle = ModelBundle::resolve(&dir).expect("complete canonical bundle resolves");
2021 assert_eq!(
2022 bundle.canonical_main.as_deref(),
2023 Some(expected_artifact.as_path())
2024 );
2025 assert!(
2026 !bundle.main.exists(),
2027 "canonical synthesis must not require the raw main checkpoint"
2028 );
2029
2030 let explicit = ModelBundle::resolve(&expected_artifact)
2031 .expect("an explicit canonical artifact resolves against its sidecars");
2032 assert_eq!(
2033 explicit.canonical_main.as_deref(),
2034 Some(expected_artifact.as_path())
2035 );
2036 }
2037}