1use crate::error::FttsError;
27use ftts_core::{
28 CancellationToken, EngineError, FrameGenerator, GenerationError, NormalizationOptions,
29 NormalizationTrace, PreparedText, SynthesisObserver, SynthesisRequest, TextPreparationError,
30 TextPreparer, TtsEngine,
31};
32use ftts_model_qwen::checkpoint::{
33 CODEC_LANGUAGE_ENGLISH_ID, CheckpointError, CodecCheckpoint, TALKER_HIDDEN, TalkerCheckpoint,
34};
35use ftts_model_qwen::generate::{QwenGenerator, QwenGeneratorConfig};
36use ftts_model_qwen::microdecoder::MicrodecoderConfig;
37use ftts_model_qwen::prompt::{CloneMode, PromptMode};
38use ftts_model_qwen::sampler::SamplingMode;
39use ftts_model_qwen::speaker::{
40 Encoder as SpeakerEncoder, SPEAKER_SAMPLE_RATE_HZ, log_mel_from_24khz_pcm,
41};
42use ftts_model_qwen::talker::TalkerConfig;
43use ftts_model_qwen::tokenizer::{QwenTokenizer, TokenizerFiles};
44use std::fs;
45use std::fs::OpenOptions;
46use std::path::{Path, PathBuf};
47use symphonia::core::audio::SampleBuffer;
48use symphonia::core::codecs::DecoderOptions;
49use symphonia::core::errors::Error as SymphoniaError;
50use symphonia::core::formats::FormatOptions;
51use symphonia::core::io::MediaSourceStream;
52use symphonia::core::meta::MetadataOptions;
53use symphonia::core::probe::Hint;
54use symphonia::default::{get_codecs, get_probe};
55
56pub const SPEAKER_VECTOR_BYTES: usize = TALKER_HIDDEN * 4;
58
59const CANONICAL_MODEL_BASENAME: &str = "qwen3-tts-12hz-0.6b-base.fttsq";
60
61fn checkpoint_error(error: CheckpointError) -> FttsError {
62 FttsError::ArtifactFormat(error.to_string())
63}
64
65#[derive(Clone, Debug)]
67pub struct ModelBundle {
68 pub root: PathBuf,
70 pub main: PathBuf,
73 pub canonical_main: Option<PathBuf>,
75 pub codec: PathBuf,
77}
78
79impl ModelBundle {
80 pub fn resolve(model: &Path) -> Result<Self, FttsError> {
88 let root = if model.is_dir() {
89 model.to_path_buf()
90 } else {
91 model
92 .parent()
93 .ok_or_else(|| {
94 FttsError::ModelNotFound(format!(
95 "model path {} has no parent directory",
96 model.display()
97 ))
98 })?
99 .to_path_buf()
100 };
101 let canonical_main = if model.is_dir() {
102 let canonical = root.join(CANONICAL_MODEL_BASENAME);
103 if canonical.is_file() {
104 Some(canonical)
105 } else {
106 None
107 }
108 } else if model.extension().and_then(|extension| extension.to_str()) == Some("fttsq") {
109 Some(model.to_path_buf())
110 } else {
111 None
112 };
113 let main = root.join("model.safetensors");
114 let codec = root.join("speech_tokenizer/model.safetensors");
115 let (main_label, main_path) = match canonical_main.as_ref() {
116 Some(path) => ("canonical talker artifact", path),
117 None => ("talker checkpoint", &main),
118 };
119 for (label, path) in [
120 (main_label, main_path),
121 ("codec checkpoint", &codec),
122 ("tokenizer vocabulary", &root.join("vocab.json")),
123 ("tokenizer merges", &root.join("merges.txt")),
124 ("tokenizer config", &root.join("tokenizer_config.json")),
125 ] {
126 if !path.is_file() {
127 return Err(FttsError::ModelNotFound(format!(
128 "{label} is missing at {}; `ftts say` needs a complete model directory \
129 ({CANONICAL_MODEL_BASENAME} or model.safetensors, \
130 speech_tokenizer/model.safetensors, \
131 vocab.json, merges.txt, tokenizer_config.json)",
132 path.display()
133 )));
134 }
135 }
136 Ok(Self {
137 root,
138 main,
139 canonical_main,
140 codec,
141 })
142 }
143}
144
145pub struct LoadedModel {
147 talker: TalkerCheckpoint,
148 codec: CodecCheckpoint,
149 tokenizer: QwenTokenizer,
150 artifact: Option<std::sync::Arc<ftts_artifacts::fttsq::MappedFttsq>>,
155}
156
157impl LoadedModel {
158 pub fn load(bundle: &ModelBundle) -> Result<Self, FttsError> {
164 let read = |name: &str| -> Result<String, FttsError> {
165 let path = bundle.root.join(name);
166 fs::read_to_string(&path).map_err(|error| {
167 FttsError::ArtifactFormat(format!("cannot read {}: {error}", path.display()))
168 })
169 };
170 let vocab = read("vocab.json")?;
171 let merges = read("merges.txt")?;
172 let config = read("tokenizer_config.json")?;
173
174 let (talker, codec, tokenizer) = std::thread::scope(|scope| {
178 let codec = scope.spawn(|| CodecCheckpoint::load(&bundle.codec));
179 let tokenizer = scope.spawn(|| {
180 QwenTokenizer::from_files_using_environment(TokenizerFiles {
181 vocab_json: &vocab,
182 merges_txt: &merges,
183 tokenizer_config_json: &config,
184 })
185 });
186 let talker = match bundle.canonical_main.as_deref() {
187 Some(path) => TalkerCheckpoint::load_fttsq_elided(
191 path,
192 ftts_model_qwen::generate::hot_elision_from_environment(),
193 ),
194 None => TalkerCheckpoint::load(&bundle.main),
195 };
196 (
197 talker,
198 codec.join().expect("codec loader panicked"),
199 tokenizer.join().expect("tokenizer builder panicked"),
200 )
201 });
202 let tokenizer = tokenizer
203 .map_err(|error| FttsError::ArtifactFormat(format!("tokenizer unusable: {error}")))?;
204
205 let talker = talker.map_err(checkpoint_error)?;
206 let artifact = talker.artifact().cloned();
209 Ok(Self {
210 talker,
211 codec: codec.map_err(checkpoint_error)?,
212 tokenizer,
213 artifact,
214 })
215 }
216}
217
218pub fn read_speaker_vector(path: &Path) -> Result<Vec<f32>, FttsError> {
228 let bytes = fs::read(path).map_err(|error| {
229 FttsError::Input(format!(
230 "cannot read speaker vector {}: {error}",
231 path.display()
232 ))
233 })?;
234 if bytes.len() != SPEAKER_VECTOR_BYTES {
235 return Err(FttsError::Input(format!(
236 "speaker vector {} is {} bytes; `ftts say --voice` expects exactly {} \
237 ({TALKER_HIDDEN} little-endian f32)",
238 path.display(),
239 bytes.len(),
240 SPEAKER_VECTOR_BYTES
241 )));
242 }
243 let vector: Vec<f32> = bytes
244 .as_chunks::<4>()
245 .0
246 .iter()
247 .map(|quad| f32::from_le_bytes(*quad))
248 .collect();
249 if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
250 return Err(FttsError::Input(format!(
251 "speaker vector {} holds a non-finite value at index {index}; it would poison every \
252 prefill position it is summed into",
253 path.display()
254 )));
255 }
256 Ok(vector)
257}
258
259#[derive(Clone, Copy, Debug)]
262pub struct DenoiseReport {
263 pub before_dbfs: f32,
265 pub after_dbfs: f32,
267}
268
269#[derive(Default)]
274pub struct ReferenceCleanup<'a> {
275 pub denoise: Option<&'a mut Option<DenoiseReport>>,
277 pub dereverb: Option<&'a mut Option<DereverbReport>>,
279}
280
281pub fn speaker_from_voice(
292 bundle: &ModelBundle,
293 path: &Path,
294 cleanup: ReferenceCleanup<'_>,
295) -> Result<Vec<f32>, FttsError> {
296 let bytes = fs::read(path).map_err(|error| {
297 FttsError::Input(format!(
298 "cannot read voice source {}: {error}",
299 path.display()
300 ))
301 })?;
302 let looks_like_audio = bytes.len() >= 12
306 && (bytes.starts_with(b"RIFF")
307 || bytes.starts_with(b"fLaC")
308 || bytes.starts_with(b"ID3")
309 || bytes.starts_with(b"OggS")
310 || &bytes[4..8] == b"ftyp");
311 if bytes.len() == SPEAKER_VECTOR_BYTES && !looks_like_audio {
312 return decode_speaker_vector(path, &bytes);
313 }
314 let pcm = decode_reference_audio_any(path)?;
315 let ReferenceCleanup { denoise, dereverb } = cleanup;
316 let pcm = match dereverb {
317 Some(report) => {
318 let before = reverb_time_s(&pcm);
319 let dried = dereverb_reference(&pcm);
320 let after = reverb_time_s(&dried);
321 if let (Some(before), Some(after)) = (before, after) {
322 *report = Some(DereverbReport {
323 before_rt60_s: before,
324 after_rt60_s: after,
325 });
326 }
327 dried
328 }
329 None => pcm,
330 };
331 let pcm = match denoise {
332 Some(report) => {
333 let before = pause_floor_dbfs(&pcm);
334 let cleaned = match neural_denoise_reference(bundle, &pcm)? {
335 Some(cleaned) => cleaned,
336 None => denoise_reference(&pcm),
337 };
338 *report = Some(DenoiseReport {
339 before_dbfs: before,
340 after_dbfs: pause_floor_dbfs(&cleaned),
341 });
342 cleaned
343 }
344 None => pcm,
345 };
346 let mel = log_mel_from_24khz_pcm(&pcm)
347 .map_err(|error| FttsError::Input(format!("cannot extract speaker features: {error}")))?;
348 let encoder = match bundle.canonical_main.as_deref() {
349 Some(artifact) => SpeakerEncoder::load_fttsq(artifact),
350 None => SpeakerEncoder::load(&bundle.main),
351 }
352 .map_err(checkpoint_error)?;
353 let vector = encoder.encode(&mel.values, mel.frames);
354 if vector.iter().all(|value| value.is_finite()) {
355 Ok(vector)
356 } else {
357 Err(FttsError::Input(
358 "speaker encoder produced a non-finite x-vector; refusing to condition synthesis"
359 .to_owned(),
360 ))
361 }
362}
363
364pub fn write_speaker_vector_new(path: &Path, vector: &[f32]) -> Result<(), FttsError> {
366 if vector.len() != TALKER_HIDDEN {
367 return Err(FttsError::Input(format!(
368 "cannot write {}-wide speaker vector; expected {TALKER_HIDDEN}",
369 vector.len()
370 )));
371 }
372 if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
373 return Err(FttsError::Input(format!(
374 "cannot write speaker vector with a non-finite value at index {index}"
375 )));
376 }
377 let mut bytes = Vec::with_capacity(SPEAKER_VECTOR_BYTES);
378 for value in vector {
379 bytes.extend_from_slice(&value.to_le_bytes());
380 }
381 use std::io::Write;
382 let mut file = OpenOptions::new()
383 .write(true)
384 .create_new(true)
385 .open(path)
386 .map_err(|error| {
387 FttsError::Input(format!(
388 "cannot create enrolled voice {} without overwriting an existing file: {error}",
389 path.display()
390 ))
391 })?;
392 file.write_all(&bytes).map_err(|error| {
393 FttsError::Input(format!(
394 "cannot write enrolled voice {}: {error}",
395 path.display()
396 ))
397 })
398}
399
400pub fn replace_speaker_vector(path: &Path, vector: &[f32]) -> Result<PathBuf, FttsError> {
410 let backup = path.with_extension("spk.bak");
411 fs::copy(path, &backup).map_err(|error| {
412 FttsError::Input(format!(
413 "cannot back up the existing voice {} to {}: {error}",
414 path.display(),
415 backup.display()
416 ))
417 })?;
418 let staging = path.with_extension("spk.incoming");
421 if staging.exists() {
422 fs::remove_file(&staging).map_err(|error| {
423 FttsError::Input(format!(
424 "cannot clear the stale staging file {}: {error}",
425 staging.display()
426 ))
427 })?;
428 }
429 write_speaker_vector_new(&staging, vector)?;
430 fs::rename(&staging, path).map_err(|error| {
431 FttsError::Input(format!(
432 "cannot replace {} with the new voice: {error}",
433 path.display()
434 ))
435 })?;
436 Ok(backup)
437}
438
439fn decode_speaker_vector(path: &Path, bytes: &[u8]) -> Result<Vec<f32>, FttsError> {
440 let vector: Vec<f32> = bytes
441 .as_chunks::<4>()
442 .0
443 .iter()
444 .map(|quad| f32::from_le_bytes(*quad))
445 .collect();
446 if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
447 return Err(FttsError::Input(format!(
448 "speaker vector {} holds a non-finite value at index {index}; it would poison every \
449 prefill position it is summed into",
450 path.display()
451 )));
452 }
453 Ok(vector)
454}
455
456const SYSTEM_DECODED_EXTENSIONS: [&str; 6] = ["m4a", "mp3", "aac", "mp4", "ogg", "opus"];
460
461pub(crate) fn private_staging_dir() -> std::io::Result<PathBuf> {
471 #[allow(deprecated)] let base = std::env::home_dir()
473 .map(|home| home.join(".cache/franken_tts"))
474 .unwrap_or_else(std::env::temp_dir);
475 let dir = base.join("staging");
476 fs::create_dir_all(&dir)?;
477 #[cfg(unix)]
478 {
479 use std::os::unix::fs::PermissionsExt;
480 fs::set_permissions(&dir, fs::Permissions::from_mode(0o700))?;
481 }
482 Ok(dir)
483}
484
485fn decode_reference_audio_any(path: &Path) -> Result<Vec<f32>, FttsError> {
491 let extension = path
492 .extension()
493 .and_then(|extension| extension.to_str())
494 .map(str::to_ascii_lowercase);
495 let needs_system_decoder = extension
496 .as_deref()
497 .is_some_and(|extension| SYSTEM_DECODED_EXTENSIONS.contains(&extension));
498 if !needs_system_decoder {
499 return decode_reference_audio(path);
500 }
501
502 let staging_dir = private_staging_dir()
503 .map_err(|error| FttsError::Generic(format!("cannot create staging directory: {error}")))?;
504 let staging = staging_dir.join(format!(
505 "ftts-enroll-{}-{}.wav",
506 std::process::id(),
507 path.file_stem()
508 .and_then(|stem| stem.to_str())
509 .unwrap_or("reference")
510 ));
511 let attempts: &[(&str, Vec<&std::ffi::OsStr>)] = &[
512 (
517 "afconvert",
518 vec![
519 "-f".as_ref(),
520 "WAVE".as_ref(),
521 "-d".as_ref(),
522 "LEI16@24000".as_ref(),
523 "-c".as_ref(),
524 "1".as_ref(),
525 path.as_os_str(),
526 staging.as_os_str(),
527 ],
528 ),
529 (
530 "ffmpeg",
531 vec![
532 "-y".as_ref(),
533 "-loglevel".as_ref(),
534 "error".as_ref(),
535 "-i".as_ref(),
536 path.as_os_str(),
537 "-acodec".as_ref(),
538 "pcm_s16le".as_ref(),
539 "-ar".as_ref(),
540 "24000".as_ref(),
541 "-ac".as_ref(),
542 "1".as_ref(),
543 staging.as_os_str(),
544 ],
545 ),
546 ];
547 let mut ran = false;
548 for (tool, arguments) in attempts {
549 match std::process::Command::new(tool).args(arguments).status() {
550 Ok(status) if status.success() => {
551 ran = true;
552 break;
553 }
554 Ok(status) => {
555 let _ = fs::remove_file(&staging);
556 return Err(FttsError::Input(format!(
557 "{tool} failed decoding reference audio {} (exit {status})",
558 path.display()
559 )));
560 }
561 Err(_) => continue, }
563 }
564 if !ran {
565 return Err(FttsError::Input(format!(
566 "reference audio {} is a compressed container and no system decoder was found; \
567 install afconvert (macOS) or ffmpeg, or supply WAV/FLAC",
568 path.display()
569 )));
570 }
571 let decoded = decode_reference_audio(&staging);
572 let _ = fs::remove_file(&staging);
573 decoded
574}
575
576fn decode_reference_audio(path: &Path) -> Result<Vec<f32>, FttsError> {
577 let file = fs::File::open(path).map_err(|error| {
578 FttsError::Input(format!(
579 "cannot open reference audio {}: {error}",
580 path.display()
581 ))
582 })?;
583 let mut hint = Hint::new();
584 if let Some(extension) = path.extension().and_then(|extension| extension.to_str()) {
585 hint.with_extension(extension);
586 }
587 let stream = MediaSourceStream::new(Box::new(file), Default::default());
588 let probed = get_probe()
589 .format(
590 &hint,
591 stream,
592 &FormatOptions::default(),
593 &MetadataOptions::default(),
594 )
595 .map_err(|error| {
596 FttsError::Input(format!(
597 "cannot identify reference audio {}: {error}",
598 path.display()
599 ))
600 })?;
601 let mut format = probed.format;
602 let track = format.default_track().ok_or_else(|| {
603 FttsError::Input(format!(
604 "reference audio {} has no default audio track",
605 path.display()
606 ))
607 })?;
608 let track_id = track.id;
609 let mut decoder = get_codecs()
610 .make(&track.codec_params, &DecoderOptions::default())
611 .map_err(|error| {
612 FttsError::Input(format!(
613 "cannot decode reference audio {}: {error}",
614 path.display()
615 ))
616 })?;
617 let mut sample_rate = None;
618 let mut mono = Vec::new();
619 loop {
620 let packet = match format.next_packet() {
621 Ok(packet) => packet,
622 Err(SymphoniaError::IoError(error))
623 if error.kind() == std::io::ErrorKind::UnexpectedEof =>
624 {
625 break;
626 }
627 Err(error) => {
628 return Err(FttsError::Input(format!(
629 "cannot read reference audio {}: {error}",
630 path.display()
631 )));
632 }
633 };
634 if packet.track_id() != track_id {
635 continue;
636 }
637 let decoded = decoder.decode(&packet).map_err(|error| {
638 FttsError::Input(format!(
639 "cannot decode reference audio {}: {error}",
640 path.display()
641 ))
642 })?;
643 let spec = *decoded.spec();
644 match sample_rate {
645 Some(rate) if rate != spec.rate => {
646 return Err(FttsError::Input(format!(
647 "reference audio {} changed sample rate mid-stream ({rate} to {} Hz)",
648 path.display(),
649 spec.rate
650 )));
651 }
652 None => sample_rate = Some(spec.rate),
653 Some(_) => {}
654 }
655 let channels = spec.channels.count();
656 let mut samples = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
657 samples.copy_interleaved_ref(decoded);
658 for frame in samples.samples().chunks_exact(channels) {
659 mono.push(frame.iter().sum::<f32>() / channels as f32);
660 }
661 }
662 let rate = sample_rate.ok_or_else(|| {
663 FttsError::Input(format!(
664 "reference audio {} contains no decodable samples",
665 path.display()
666 ))
667 })?;
668 if mono.is_empty() {
669 return Err(FttsError::Input(format!(
670 "reference audio {} contains no PCM samples",
671 path.display()
672 )));
673 }
674 let pcm = resample_to_speaker_rate(mono, rate);
675 if pcm.is_empty() {
679 return Err(FttsError::Input(format!(
680 "reference audio {} is too short to resample from {rate} Hz to \
681 {SPEAKER_SAMPLE_RATE_HZ} Hz; supply a longer recording",
682 path.display()
683 )));
684 }
685 Ok(pcm)
686}
687
688fn resample_to_speaker_rate(mono: Vec<f32>, from_rate: u32) -> Vec<f32> {
702 if from_rate == SPEAKER_SAMPLE_RATE_HZ {
703 return mono;
704 }
705 resample_lanczos(&mono, from_rate, SPEAKER_SAMPLE_RATE_HZ)
706}
707
708fn resample_lanczos(mono: &[f32], from_rate: u32, to_rate: u32) -> Vec<f32> {
710 if from_rate == to_rate {
711 return mono.to_vec();
712 }
713 const LOBES: f64 = 6.0;
718 let ratio = f64::from(to_rate) / f64::from(from_rate);
719 let cutoff = ratio.min(1.0);
720 let half = (LOBES / cutoff).ceil() as isize;
721 let out_len = ((mono.len() as f64) * ratio).round() as usize;
722
723 let mut out = Vec::with_capacity(out_len);
724 for index in 0..out_len {
725 let center = index as f64 / ratio;
726 let first = center.floor() as isize - half + 1;
727 let mut acc = 0.0_f64;
728 let mut norm = 0.0_f64;
729 for tap in first..first + 2 * half {
730 if tap < 0 {
731 continue;
732 }
733 let Some(sample) = mono.get(tap as usize) else {
734 break;
735 };
736 let weight = lanczos_tap(center - tap as f64, cutoff, LOBES);
737 acc += weight * f64::from(*sample);
738 norm += weight;
739 }
740 out.push(if norm.abs() > 1e-12 {
741 (acc / norm) as f32
742 } else {
743 0.0
744 });
745 }
746 out
747}
748
749const DENOISE_FRAME: usize = 512;
752
753const DENOISE_HOP: usize = DENOISE_FRAME / 4;
756
757const DD_ALPHA: f32 = 0.92;
760
761const OMLSA_GAIN_FLOOR: f32 = 0.017_782_79;
769
770const NOISE_BLOCK_FRAMES: usize = 256;
773
774const SPEECH_ABSENCE_PRIOR: f32 = 0.6;
777
778#[allow(clippy::excessive_precision)]
785const NOISE_QUANTILE_BIAS: f32 = 9.491_221; const NOISE_INIT_QUANTILE: f32 = 0.1;
790
791pub const DENOISE_ARTIFACT_RELPATH: &str = "denoise/fastenhancer-s-48k.safetensors";
840
841fn neural_denoise_reference(
857 bundle: &ModelBundle,
858 pcm24k: &[f32],
859) -> Result<Option<Vec<f32>>, FttsError> {
860 let path = bundle.root.join(DENOISE_ARTIFACT_RELPATH);
861 if !path.is_file() {
862 return Ok(None);
863 }
864 if std::env::var("FTTS_DENOISE_ENGINE").is_ok_and(|v| v.eq_ignore_ascii_case("omlsa")) {
865 return Ok(None);
866 }
867 let enhancer = ftts_artifacts::enhance_loader::open_enhancer(&path).map_err(|error| {
868 FttsError::ArtifactFormat(format!(
869 "denoiser artifact {} is unreadable ({error}); re-fetch it with `ftts pull --force`",
870 path.display()
871 ))
872 })?;
873 Ok(Some(enhancer.enhance_24k(pcm24k)))
874}
875
876fn denoise_reference(pcm: &[f32]) -> Vec<f32> {
877 const DENOISE_MIN_FRAMES: usize = 32;
881 if pcm.len() < DENOISE_FRAME + (DENOISE_MIN_FRAMES - 1) * DENOISE_HOP {
882 return pcm.to_vec();
883 }
884 let mut planner = rustfft::FftPlanner::<f32>::new();
885 let forward = planner.plan_fft_forward(DENOISE_FRAME);
886 let inverse = planner.plan_fft_inverse(DENOISE_FRAME);
887
888 let window: Vec<f32> = (0..DENOISE_FRAME)
889 .map(|n| {
890 let phase = std::f32::consts::TAU * n as f32 / DENOISE_FRAME as f32;
891 0.5 - 0.5 * phase.cos()
892 })
893 .collect();
894
895 let bins = DENOISE_FRAME / 2 + 1;
896 let starts: Vec<usize> = (0..=pcm.len() - DENOISE_FRAME)
897 .step_by(DENOISE_HOP)
898 .collect();
899
900 let mut powers: Vec<Vec<f32>> = Vec::with_capacity(starts.len());
903 let mut scratch: Vec<rustfft::num_complex::Complex<f32>> =
904 vec![rustfft::num_complex::Complex::new(0.0, 0.0); DENOISE_FRAME];
905 for &start in &starts {
906 for (slot, n) in scratch.iter_mut().zip(0..DENOISE_FRAME) {
907 *slot = rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0);
908 }
909 forward.process(&mut scratch);
910 powers.push((0..bins).map(|bin| scratch[bin].norm_sqr()).collect());
911 }
912
913 let blocks = powers.len().div_ceil(NOISE_BLOCK_FRAMES);
928 let mut noise = vec![f32::INFINITY; bins];
929 let mut column: Vec<f32> = Vec::with_capacity(NOISE_BLOCK_FRAMES);
930 for block in 0..blocks {
931 let span = block * NOISE_BLOCK_FRAMES..((block + 1) * NOISE_BLOCK_FRAMES).min(powers.len());
932 for (bin, slot) in noise.iter_mut().enumerate() {
933 column.clear();
934 column.extend(powers[span.clone()].iter().map(|frame| frame[bin]));
935 column.sort_by(f32::total_cmp);
936 let rank = ((column.len() as f32 - 1.0) * NOISE_INIT_QUANTILE).round() as usize;
937 *slot = slot.min(column[rank].max(1e-12) * NOISE_QUANTILE_BIAS);
938 }
939 }
940
941 let mut prev_gain = vec![1.0_f32; bins];
943 let mut prev_gamma = vec![1.0_f32; bins];
944
945 let mut out = vec![0.0_f32; pcm.len()];
946 let mut weight = vec![0.0_f32; pcm.len()];
947
948 for (index, &start) in starts.iter().enumerate() {
949 let mut frame: Vec<rustfft::num_complex::Complex<f32>> = (0..DENOISE_FRAME)
950 .map(|n| rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0))
951 .collect();
952 forward.process(&mut frame);
953 let power = &powers[index];
954
955 for bin in 0..bins {
956 let gamma = (power[bin] / noise[bin]).min(1e6);
957 let xi = (DD_ALPHA * prev_gain[bin].powi(2) * prev_gamma[bin]
958 + (1.0 - DD_ALPHA) * (gamma - 1.0).max(0.0))
959 .max(1e-6);
960
961 let nu = (xi / (1.0 + xi)) * gamma;
962 let lsa =
963 ((xi / (1.0 + xi)) * (0.5 * exponential_integral_e1(nu)).exp()).clamp(0.0, 1.0);
964
965 let odds = SPEECH_ABSENCE_PRIOR / (1.0 - SPEECH_ABSENCE_PRIOR);
969 let presence = 1.0 / (1.0 + odds * (1.0 + xi) * (-nu).exp());
970 let presence = presence.clamp(0.0, 1.0);
971
972 let gain = (lsa.max(OMLSA_GAIN_FLOOR).powf(presence)
973 * OMLSA_GAIN_FLOOR.powf(1.0 - presence))
974 .clamp(OMLSA_GAIN_FLOOR, 1.0);
975
976 prev_gain[bin] = gain;
977 prev_gamma[bin] = gamma;
978
979 frame[bin] *= gain;
980 let mirror = DENOISE_FRAME - bin;
981 if mirror != bin && mirror < DENOISE_FRAME {
984 frame[mirror] *= gain;
985 }
986 }
987
988 inverse.process(&mut frame);
989 let scale = 1.0 / DENOISE_FRAME as f32;
990 for n in 0..DENOISE_FRAME {
991 out[start + n] += frame[n].re * scale * window[n];
992 weight[start + n] += window[n] * window[n];
993 }
994 }
995
996 const WOLA_MIN_WEIGHT: f32 = 0.15;
1002 for (sample, energy) in out.iter_mut().zip(weight.iter()) {
1003 if *energy > WOLA_MIN_WEIGHT {
1004 *sample /= *energy;
1005 }
1006 }
1007 let covered =
1010 starts.first().copied().unwrap_or(0)..starts.last().map_or(0, |last| last + DENOISE_FRAME);
1011 for (index, sample) in out.iter_mut().enumerate() {
1012 if !covered.contains(&index) || weight[index] <= WOLA_MIN_WEIGHT {
1013 *sample = pcm[index];
1014 }
1015 }
1016 out
1017}
1018
1019fn exponential_integral_e1(x: f32) -> f32 {
1026 if x <= 0.0 {
1027 return 0.0;
1030 }
1031 let x = f64::from(x);
1032 let value = if x < 1.0 {
1033 const A: [f64; 6] = [
1035 -0.577_215_664_9,
1036 0.999_991_93,
1037 -0.249_910_55,
1038 0.055_199_68,
1039 -0.009_760_04,
1040 0.001_078_57,
1041 ];
1042 let mut acc = 0.0;
1043 for (power, coefficient) in A.iter().enumerate() {
1044 acc += coefficient * x.powi(power as i32);
1045 }
1046 acc - x.ln()
1047 } else {
1048 const A: [f64; 4] = [8.573_328_74, 18.059_016_97, 8.634_760_89, 0.267_773_734];
1050 const B: [f64; 4] = [9.573_322_34, 25.632_956_15, 21.099_653_08, 3.958_496_93];
1051 let numerator = x.powi(4) + A[0] * x.powi(3) + A[1] * x * x + A[2] * x + A[3];
1052 let denominator = x.powi(4) + B[0] * x.powi(3) + B[1] * x * x + B[2] * x + B[3];
1053 (numerator / denominator) / (x * x.exp())
1054 };
1055 value as f32
1056}
1057
1058const DEREVERB_FRAME: usize = 1024;
1067const DEREVERB_HOP: usize = 256;
1068
1069const DEREVERB_TAPS: usize = 40;
1071
1072const DEREVERB_DELAY: usize = 2;
1076
1077const DEREVERB_ITERATIONS: usize = 3;
1081
1082const DEREVERB_LOADING: f64 = 1e-4;
1085
1086#[derive(Clone, Copy, Debug)]
1088pub struct DereverbReport {
1089 pub before_rt60_s: f32,
1091 pub after_rt60_s: f32,
1093}
1094
1095fn dereverb_reference(pcm: &[f32]) -> Vec<f32> {
1127 if pcm.len() < DEREVERB_FRAME * 4 {
1128 return pcm.to_vec();
1129 }
1130 let mut planner = rustfft::FftPlanner::<f32>::new();
1131 let forward = planner.plan_fft_forward(DEREVERB_FRAME);
1132 let inverse = planner.plan_fft_inverse(DEREVERB_FRAME);
1133
1134 let window: Vec<f32> = (0..DEREVERB_FRAME)
1135 .map(|n| {
1136 let phase = std::f32::consts::TAU * n as f32 / DEREVERB_FRAME as f32;
1137 0.5 - 0.5 * phase.cos()
1138 })
1139 .collect();
1140
1141 let bins = DEREVERB_FRAME / 2 + 1;
1142 let starts: Vec<usize> = (0..=pcm.len() - DEREVERB_FRAME)
1143 .step_by(DEREVERB_HOP)
1144 .collect();
1145 let frames = starts.len();
1146 if frames <= DEREVERB_DELAY + DEREVERB_TAPS + 2 {
1147 return pcm.to_vec();
1148 }
1149
1150 let mut observed: Vec<Vec<Complex64>> = Vec::with_capacity(frames);
1152 let mut scratch: Vec<rustfft::num_complex::Complex<f32>> =
1153 vec![rustfft::num_complex::Complex::new(0.0, 0.0); DEREVERB_FRAME];
1154 for &start in &starts {
1155 for (slot, n) in scratch.iter_mut().zip(0..DEREVERB_FRAME) {
1156 *slot = rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0);
1157 }
1158 forward.process(&mut scratch);
1159 observed.push(
1160 scratch[..bins]
1161 .iter()
1162 .map(|value| Complex64::new(f64::from(value.re), f64::from(value.im)))
1163 .collect(),
1164 );
1165 }
1166
1167 let mut desired = observed.clone();
1168 for _ in 0..DEREVERB_ITERATIONS {
1169 for bin in 0..bins {
1170 let mut power: Vec<f64> = (0..frames).map(|t| desired[t][bin].norm_sqr()).collect();
1173 let mean = power.iter().sum::<f64>() / frames as f64;
1174 let floor = (mean * 1e-6).max(1e-12);
1175 for value in &mut power {
1176 *value = value.max(floor);
1177 }
1178
1179 let taps = DEREVERB_TAPS;
1180 let mut covariance = vec![Complex64::new(0.0, 0.0); taps * taps];
1181 let mut cross = vec![Complex64::new(0.0, 0.0); taps];
1182 for t in (DEREVERB_DELAY + taps)..frames {
1183 let weight = 1.0 / power[t];
1184 let regressor: Vec<Complex64> = (0..taps)
1186 .map(|lag| observed[t - DEREVERB_DELAY - lag][bin])
1187 .collect();
1188 for row in 0..taps {
1189 let scaled = regressor[row] * weight;
1190 for column in row..taps {
1191 covariance[row * taps + column] += scaled * regressor[column].conj();
1192 }
1193 cross[row] += scaled * observed[t][bin].conj();
1194 }
1195 }
1196 for row in 0..taps {
1198 for column in 0..row {
1199 covariance[row * taps + column] = covariance[column * taps + row].conj();
1200 }
1201 }
1202 let trace: f64 = (0..taps).map(|i| covariance[i * taps + i].re).sum();
1203 if trace <= 0.0 {
1204 continue;
1205 }
1206 let loading = trace / taps as f64 * DEREVERB_LOADING;
1207 for i in 0..taps {
1208 covariance[i * taps + i] += Complex64::new(loading, 0.0);
1209 }
1210
1211 let Some(filter) = solve_complex_system(&mut covariance, &mut cross, taps) else {
1212 continue;
1213 };
1214 for t in 0..frames {
1215 if t < DEREVERB_DELAY + taps {
1216 desired[t][bin] = observed[t][bin];
1217 continue;
1218 }
1219 let mut tail = Complex64::new(0.0, 0.0);
1220 for (lag, coefficient) in filter.iter().enumerate() {
1221 tail += coefficient.conj() * observed[t - DEREVERB_DELAY - lag][bin];
1222 }
1223 desired[t][bin] = observed[t][bin] - tail;
1224 }
1225 }
1226 }
1227
1228 let mut out = vec![0.0_f32; pcm.len()];
1231 let mut weight = vec![0.0_f32; pcm.len()];
1232 for (index, &start) in starts.iter().enumerate() {
1233 let mut frame = vec![rustfft::num_complex::Complex::new(0.0_f32, 0.0); DEREVERB_FRAME];
1234 for bin in 0..bins {
1235 let value = desired[index][bin];
1236 #[allow(clippy::cast_possible_truncation)]
1237 let value = rustfft::num_complex::Complex::new(value.re as f32, value.im as f32);
1238 frame[bin] = value;
1239 let mirror = DEREVERB_FRAME - bin;
1240 if mirror != bin && mirror < DEREVERB_FRAME {
1241 frame[mirror] = value.conj();
1242 }
1243 }
1244 inverse.process(&mut frame);
1245 let scale = 1.0 / DEREVERB_FRAME as f32;
1246 for n in 0..DEREVERB_FRAME {
1247 out[start + n] += frame[n].re * scale * window[n];
1248 weight[start + n] += window[n] * window[n];
1249 }
1250 }
1251 for (sample, energy) in out.iter_mut().zip(weight.iter()) {
1252 if *energy > 1e-6 {
1253 *sample /= *energy;
1254 }
1255 }
1256 let covered =
1257 starts.first().copied().unwrap_or(0)..starts.last().map_or(0, |last| last + DEREVERB_FRAME);
1258 for (index, sample) in out.iter_mut().enumerate() {
1259 if !covered.contains(&index) || weight[index] <= 1e-6 {
1260 *sample = pcm[index];
1261 }
1262 }
1263 out
1264}
1265
1266type Complex64 = rustfft::num_complex::Complex<f64>;
1271
1272fn solve_complex_system(
1277 a: &mut [Complex64],
1278 b: &mut [Complex64],
1279 n: usize,
1280) -> Option<Vec<Complex64>> {
1281 for column in 0..n {
1282 let (pivot, magnitude) = (column..n).fold((column, 0.0_f64), |best, row| {
1283 let candidate = a[row * n + column].norm_sqr();
1284 if candidate > best.1 {
1285 (row, candidate)
1286 } else {
1287 best
1288 }
1289 });
1290 if magnitude <= f64::MIN_POSITIVE {
1291 return None;
1292 }
1293 if pivot != column {
1294 for k in 0..n {
1295 a.swap(pivot * n + k, column * n + k);
1296 }
1297 b.swap(pivot, column);
1298 }
1299 let diagonal = a[column * n + column];
1300 for row in (column + 1)..n {
1301 let factor = a[row * n + column] / diagonal;
1302 if factor == Complex64::new(0.0, 0.0) {
1303 continue;
1304 }
1305 for k in column..n {
1306 let value = a[column * n + k] * factor;
1307 a[row * n + k] -= value;
1308 }
1309 let value = b[column] * factor;
1310 b[row] -= value;
1311 }
1312 }
1313 let mut solution = vec![Complex64::new(0.0, 0.0); n];
1314 for row in (0..n).rev() {
1315 let mut accumulator = b[row];
1316 for k in (row + 1)..n {
1317 accumulator -= a[row * n + k] * solution[k];
1318 }
1319 solution[row] = accumulator / a[row * n + row];
1320 }
1321 Some(solution)
1322}
1323
1324fn reverb_time_s(pcm: &[f32]) -> Option<f32> {
1331 let hop = (SPEAKER_SAMPLE_RATE_HZ as usize) / 100; if pcm.len() < hop * 32 {
1333 return None;
1334 }
1335 let envelope: Vec<f32> = pcm
1336 .chunks_exact(hop)
1337 .map(|chunk| {
1338 let energy = chunk.iter().map(|s| s * s).sum::<f32>() / chunk.len() as f32;
1339 10.0 * (energy + 1e-9).log10()
1340 })
1341 .collect();
1342 let peak = envelope.iter().copied().fold(f32::NEG_INFINITY, f32::max);
1343 let span = 15_usize; let mut slopes: Vec<f32> = Vec::new();
1345 for index in 1..envelope.len().saturating_sub(span) {
1346 if envelope[index] < peak - 25.0 || envelope[index] <= envelope[index - 1] {
1347 continue;
1348 }
1349 let drop = envelope[index] - envelope[index + span - 1];
1350 if drop < 6.0 {
1351 continue;
1352 }
1353 slopes.push(drop / (span as f32 * 0.01));
1354 }
1355 if slopes.is_empty() {
1356 return None;
1357 }
1358 slopes.sort_by(f32::total_cmp);
1359 let median = slopes[slopes.len() / 2];
1360 (median > 0.0).then(|| 60.0 / median)
1361}
1362
1363fn pause_floor_dbfs(pcm: &[f32]) -> f32 {
1368 let span = (SPEAKER_SAMPLE_RATE_HZ as usize) / 20;
1369 if pcm.len() < span {
1370 return f32::NEG_INFINITY;
1371 }
1372 let mut windows: Vec<f32> = pcm
1373 .chunks_exact(span)
1374 .map(|chunk| {
1375 (chunk
1376 .iter()
1377 .map(|s| f64::from(*s) * f64::from(*s))
1378 .sum::<f64>()
1379 / chunk.len() as f64)
1380 .sqrt() as f32
1381 })
1382 .filter(|rms| *rms > 0.0)
1383 .collect();
1384 if windows.is_empty() {
1385 return f32::NEG_INFINITY;
1386 }
1387 windows.sort_by(f32::total_cmp);
1388 let keep = (windows.len() / 10).max(1);
1389 let mean = windows[..keep].iter().sum::<f32>() / keep as f32;
1390 20.0 * mean.log10()
1391}
1392
1393fn lanczos_tap(offset: f64, cutoff: f64, lobes: f64) -> f64 {
1395 let scaled = cutoff * offset;
1396 if scaled.abs() >= lobes {
1397 return 0.0;
1398 }
1399 sinc(scaled) * sinc(scaled / lobes)
1400}
1401
1402fn sinc(x: f64) -> f64 {
1404 if x.abs() < 1e-12 {
1405 return 1.0;
1406 }
1407 let scaled = std::f64::consts::PI * x;
1408 scaled.sin() / scaled
1409}
1410
1411struct PreparedPassThrough {
1413 prepared: PreparedText,
1414}
1415
1416impl TextPreparer for PreparedPassThrough {
1417 fn prepare(
1418 &self,
1419 _text: &str,
1420 _options: &NormalizationOptions,
1421 ) -> Result<PreparedText, TextPreparationError> {
1422 Ok(PreparedText::new(
1423 self.prepared.token_ids.clone(),
1424 NormalizationTrace {
1425 mode: self.prepared.normalization_trace.mode,
1426 unicode_version: self.prepared.normalization_trace.unicode_version.clone(),
1427 changes: self.prepared.normalization_trace.changes.clone(),
1428 },
1429 ))
1430 }
1431}
1432
1433pub struct SynthesizedAudio {
1435 pub frames: u64,
1437 pub prepared_token_count: usize,
1439 pub pcm: Vec<f32>,
1441 pub ttfa: Option<std::time::Duration>,
1446}
1447
1448#[allow(clippy::too_many_arguments)]
1455pub fn synthesize(
1456 model: &LoadedModel,
1457 engine: &TtsEngine,
1458 request: &SynthesisRequest,
1459 speaker: &[f32],
1460 seed: u64,
1461 cancellation: &CancellationToken,
1462 observer: &dyn SynthesisObserver,
1463) -> Result<SynthesizedAudio, FttsError> {
1464 let prepared_raw = model
1466 .tokenizer
1467 .prepare(&request.text, &request.normalization_options)
1468 .map_err(|error| FttsError::Input(format!("text preparation failed: {error}")))?;
1469 let wrapped = TalkerCheckpoint::wrap_target_ids(&prepared_raw.token_ids);
1470 let prepared = PreparedText::new(wrapped.clone(), prepared_raw.normalization_trace);
1471
1472 let ids = TalkerCheckpoint::utterance_text_ids(&wrapped);
1474 let table = model
1475 .talker
1476 .gather_text_rows(&ids)
1477 .map_err(checkpoint_error)?;
1478
1479 let header = model
1481 .talker
1482 .xvector_header(&table, speaker, CODEC_LANGUAGE_ENGLISH_ID)
1483 .map_err(checkpoint_error)?;
1484 let tts_eos = model.talker.tts_eos(&table);
1485
1486 let talker_layers = model.talker.talker_layer_weights();
1488 let micro_layers = model.talker.microdecoder_layer_weights();
1489 let residual = model.talker.residual_embedding_slices();
1490 let heads = model.talker.microdecoder_head_slices();
1491 let micro_residual = &residual[..residual.len() - 1];
1494
1495 let mut generator = QwenGenerator::new_with_artifact(
1496 QwenGeneratorConfig {
1497 talker_config: TalkerConfig::default(),
1498 talker_weights: model.talker.talker_weights(&talker_layers),
1499 text: model.talker.text_weights(&table),
1500 feedback: model.talker.feedback_tables(&residual),
1501 microdecoder_config: MicrodecoderConfig::default(),
1502 microdecoder_weights: model.talker.microdecoder_weights(
1503 µ_layers,
1504 micro_residual,
1505 &heads,
1506 ),
1507 prompt_mode: PromptMode {
1508 clone_mode: CloneMode::XVector,
1509 non_streaming_mode: false,
1510 },
1511 header,
1512 tts_eos,
1513 reference: None,
1514 sampling_mode: SamplingMode::Production,
1525 seed,
1526 },
1527 model.artifact.as_deref(),
1528 );
1529
1530 let preparer = PreparedPassThrough { prepared };
1539 let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<ftts_core::CodeFrame>(256);
1540 let codec = &model.codec;
1541 let synthesis_started = std::time::Instant::now();
1542 let (result, pcm, ttfa) = std::thread::scope(
1543 |scope| -> Result<
1544 (
1545 ftts_core::SynthesisResult,
1546 Vec<f32>,
1547 Option<std::time::Duration>,
1548 ),
1549 FttsError,
1550 > {
1551 let worker = scope.spawn(
1552 move || -> Result<(Vec<f32>, Option<std::time::Duration>), FttsError> {
1553 ftts_kernels::team::bypass_team_on_this_thread();
1556 const PACKET_FRAMES: usize = 4;
1557 let mut state = codec.stream_state();
1558 let mut pcm = Vec::new();
1559 let mut packet_pcm = Vec::new();
1562 let mut packet: Vec<i32> = Vec::with_capacity(16 * PACKET_FRAMES);
1563 let mut packet_frames = 0_usize;
1564 let mut first_audio_at: Option<std::time::Duration> = None;
1565 while let Ok(frame) = frame_rx.recv() {
1566 if frame.codes.len() != 16 {
1567 return Err(FttsError::Generic(format!(
1568 "generated frame carries {} codes, expected 16",
1569 frame.codes.len()
1570 )));
1571 }
1572 for code in &frame.codes {
1573 packet.push(i32::try_from(*code).map_err(|_| {
1574 FttsError::Generic(format!(
1575 "generated code {code} does not fit the codec's i32"
1576 ))
1577 })?);
1578 }
1579 packet_frames += 1;
1580 if packet_frames == PACKET_FRAMES {
1581 codec
1582 .stream_push(&mut state, &packet, packet_frames, &mut packet_pcm)
1583 .map_err(checkpoint_error)?;
1584 pcm.extend_from_slice(&packet_pcm);
1585 first_audio_at.get_or_insert_with(|| synthesis_started.elapsed());
1586 packet.clear();
1587 packet_frames = 0;
1588 }
1589 }
1590 if packet_frames > 0 {
1591 codec
1592 .stream_push(&mut state, &packet, packet_frames, &mut packet_pcm)
1593 .map_err(checkpoint_error)?;
1594 pcm.extend_from_slice(&packet_pcm);
1595 first_audio_at.get_or_insert_with(|| synthesis_started.elapsed());
1596 }
1597 Ok((pcm, first_audio_at))
1598 },
1599 );
1600
1601 let mut tee = TeeGenerator {
1602 inner: &mut generator,
1603 frames: frame_tx,
1604 };
1605 let result = engine
1606 .synthesize(
1607 request.clone(),
1608 &preparer,
1609 &mut tee as &mut dyn FrameGenerator,
1610 cancellation,
1611 observer,
1612 )
1613 .map_err(engine_error);
1614 drop(tee); let pcm = worker.join().expect("codec worker must not panic");
1616 let result = result?;
1620 let (pcm, ttfa) = pcm?;
1621 Ok((result, pcm, ttfa))
1622 },
1623 )?;
1624
1625 if result.code_frames.is_empty() {
1626 return Err(FttsError::Generic(
1627 "the talker stopped before emitting a frame; there is no audio to write. This is a \
1628 model or prompt problem, not an output problem — check the speaker vector and the \
1629 text"
1630 .to_owned(),
1631 ));
1632 }
1633
1634 Ok(SynthesizedAudio {
1635 frames: result.generated_frames,
1636 prepared_token_count: result.prepared_token_count,
1637 pcm,
1638 ttfa,
1639 })
1640}
1641
1642struct TeeGenerator<'a> {
1648 inner: &'a mut dyn FrameGenerator,
1649 frames: std::sync::mpsc::SyncSender<ftts_core::CodeFrame>,
1650}
1651
1652impl FrameGenerator for TeeGenerator<'_> {
1653 fn begin_utterance(&mut self, prepared: &PreparedText) -> Result<(), GenerationError> {
1654 self.inner.begin_utterance(prepared)
1655 }
1656
1657 fn next_frame(&mut self) -> Result<Option<ftts_core::CodeFrame>, GenerationError> {
1658 let frame = self.inner.next_frame()?;
1659 if let Some(frame) = &frame
1660 && self.frames.send(frame.clone()).is_err()
1661 {
1662 return Err(GenerationError::new(
1663 "the codec worker stopped accepting frames; its error follows at join",
1664 ));
1665 }
1666 Ok(frame)
1667 }
1668}
1669
1670fn engine_error(error: EngineError) -> FttsError {
1672 match error {
1673 EngineError::BudgetExceeded(_) => FttsError::BudgetTimeout(error.to_string()),
1674 EngineError::ResourceAdmission(_) => FttsError::BudgetTimeout(error.to_string()),
1675 EngineError::TextPreparation(_) => FttsError::Input(error.to_string()),
1676 other => FttsError::Generic(other.to_string()),
1677 }
1678}
1679
1680#[must_use]
1682pub fn generation_error(message: &str) -> GenerationError {
1683 GenerationError::new(message)
1684}
1685
1686#[cfg(test)]
1687mod tests {
1688 use super::*;
1689
1690 #[test]
1693 fn audio_at_the_pinned_rate_is_returned_bit_for_bit() {
1694 let pcm: Vec<f32> = (0..4_096)
1695 .map(|n| (n as f32 * 0.017).sin() * 0.4 + (n as f32 * 0.31).sin() * 0.05)
1696 .collect();
1697 let out = resample_to_speaker_rate(pcm.clone(), SPEAKER_SAMPLE_RATE_HZ);
1698 assert_eq!(out.len(), pcm.len());
1699 for (index, (a, b)) in out.iter().zip(pcm.iter()).enumerate() {
1700 assert!(
1701 a.to_bits() == b.to_bits(),
1702 "sample {index} was altered at the pinned rate"
1703 );
1704 }
1705 }
1706
1707 #[test]
1715 fn the_exponential_integral_matches_its_series_expansion() {
1716 for (x, expected) in [
1718 (0.1_f32, 1.822_923_9_f32),
1719 (0.5, 0.559_773_6),
1720 (0.9, 0.260_183_94),
1721 ] {
1722 let actual = exponential_integral_e1(x);
1723 let relative = ((actual - expected) / expected).abs();
1724 assert!(
1725 relative < 1e-6,
1726 "E1({x}) = {actual} but the series gives {expected} (relative {relative:e})"
1727 );
1728 }
1729
1730 let below = exponential_integral_e1(0.999_9);
1732 let above = exponential_integral_e1(1.000_1);
1733 assert!(
1734 below > above && (below - above).abs() < 1e-4,
1735 "the series and rational branches disagree across x = 1: {below} vs {above}"
1736 );
1737 assert_eq!(
1738 exponential_integral_e1(0.0),
1739 0.0,
1740 "a non-positive argument must not produce NaN"
1741 );
1742 }
1743
1744 #[test]
1748 fn denoise_lowers_the_floor_between_bursts_without_eating_the_signal() {
1749 const TONE_HZ: f64 = 700.0;
1750 let samples = SPEAKER_SAMPLE_RATE_HZ as usize * 2;
1751 let mut state = 0x2545_F491_4F6C_DD1D_u64;
1753 let mut noise = || {
1754 state ^= state << 13;
1755 state ^= state >> 7;
1756 state ^= state << 17;
1757 ((state >> 40) as f32 / 16_777_216.0) - 0.5
1758 };
1759
1760 let clean: Vec<f32> = (0..samples)
1762 .map(|n| {
1763 let t = n as f64 / f64::from(SPEAKER_SAMPLE_RATE_HZ);
1764 let speaking = (n / (SPEAKER_SAMPLE_RATE_HZ as usize / 2)).is_multiple_of(2);
1765 if speaking {
1766 (std::f64::consts::TAU * TONE_HZ * t).sin() as f32 * 0.35
1767 } else {
1768 0.0
1769 }
1770 })
1771 .collect();
1772 let noisy: Vec<f32> = clean.iter().map(|s| s + noise() * 0.1).collect();
1776
1777 let cleaned = denoise_reference(&noisy);
1778 assert_eq!(cleaned.len(), noisy.len(), "denoise must preserve length");
1779 assert!(
1780 cleaned.iter().all(|s| s.is_finite()),
1781 "denoise produced a non-finite sample"
1782 );
1783
1784 let before = pause_floor_dbfs(&noisy);
1785 let after = pause_floor_dbfs(&cleaned);
1786 assert!(
1787 after < before - 3.0,
1788 "expected the pause floor to drop by >3 dB, got {before:.1} -> {after:.1} dBFS"
1789 );
1790
1791 let span = SPEAKER_SAMPLE_RATE_HZ as usize / 4;
1793 let peak_rms = |pcm: &[f32]| {
1794 pcm.chunks_exact(span)
1795 .map(|c| (c.iter().map(|s| s * s).sum::<f32>() / c.len() as f32).sqrt())
1796 .fold(0.0_f32, f32::max)
1797 };
1798 let kept = peak_rms(&cleaned) / peak_rms(&noisy);
1799 assert!(
1800 kept > 0.7,
1801 "denoise removed too much of the signal: peak RMS kept {kept:.3} of the original"
1802 );
1803 }
1804
1805 #[test]
1813 fn denoise_keeps_a_reference_that_opens_on_speech() {
1814 let rate = SPEAKER_SAMPLE_RATE_HZ as usize;
1815 let mut state = 0x9E37_79B9_7F4A_7C15_u64;
1816 let mut hiss = || {
1817 state ^= state << 13;
1818 state ^= state >> 7;
1819 state ^= state << 17;
1820 ((state >> 40) as f32 / 16_777_216.0) - 0.5
1821 };
1822 let burst = rate / 4;
1823 let noisy: Vec<f32> = (0..rate * 3)
1824 .map(|n| {
1825 let t = n as f64 / rate as f64;
1826 let index = n / burst;
1827 let voice = if index.is_multiple_of(2) {
1828 let vibrato = 1.0 + 0.03 * (std::f64::consts::TAU * 5.5 * t).sin();
1831 let phase = (n % burst) as f32 / burst as f32;
1832 let envelope = (std::f32::consts::PI * phase).sin();
1833 let f0 = 140.0 * vibrato * (1.0 + 0.15 * (index / 2) as f64);
1834 (1..=10)
1835 .map(|h| {
1836 let a = 0.3 / h as f32;
1837 (std::f64::consts::TAU * f0 * h as f64 * t).sin() as f32 * a
1838 })
1839 .sum::<f32>()
1840 * envelope
1841 } else {
1842 0.0
1843 };
1844 voice + hiss() * 0.02
1845 })
1846 .collect();
1847
1848 let cleaned = denoise_reference(&noisy);
1849 let rms = |pcm: &[f32]| (pcm.iter().map(|s| s * s).sum::<f32>() / pcm.len() as f32).sqrt();
1850 let kept = rms(&cleaned[..burst]) / rms(&noisy[..burst]);
1851 assert!(
1852 kept > 0.7,
1853 "the opening burst kept only {kept:.3} of its energy; the noise floor is being \
1854 seeded from speech the estimator has not yet learned to exclude"
1855 );
1856 }
1857
1858 #[test]
1868 fn denoise_does_not_preferentially_zap_high_frequencies() {
1869 let rate = SPEAKER_SAMPLE_RATE_HZ as usize;
1870 let mut state = 0xDEAD_BEEF_1234_5678_u64;
1871 let mut hiss = || {
1872 state ^= state << 13;
1873 state ^= state >> 7;
1874 state ^= state << 17;
1875 ((state >> 40) as f32 / 16_777_216.0) - 0.5
1876 };
1877 let probes: [f64; 5] = [300.0, 1_200.0, 3_000.0, 6_000.0, 9_000.0];
1880 let burst = rate / 4;
1881 let noisy: Vec<f32> = (0..rate * 3)
1882 .map(|n| {
1883 let t = n as f64 / rate as f64;
1884 let voice = if (n / burst).is_multiple_of(2) {
1885 let phase = (n % burst) as f32 / burst as f32;
1886 let envelope = (std::f32::consts::PI * phase).sin();
1887 probes
1888 .iter()
1889 .map(|hz| (std::f64::consts::TAU * hz * t).sin() as f32 * 0.12)
1890 .sum::<f32>()
1891 * envelope
1892 } else {
1893 0.0
1894 };
1895 voice + hiss() * 0.02
1896 })
1897 .collect();
1898
1899 let cleaned = denoise_reference(&noisy);
1900
1901 let span = burst / 2..burst;
1904 let energy_at = |pcm: &[f32], hz: f64| -> f32 {
1905 let (mut re, mut im) = (0.0_f64, 0.0_f64);
1906 for (offset, sample) in pcm[span.clone()].iter().enumerate() {
1907 let t = (span.start + offset) as f64 / rate as f64;
1908 let angle = std::f64::consts::TAU * hz * t;
1909 re += f64::from(*sample) * angle.cos();
1910 im += f64::from(*sample) * angle.sin();
1911 }
1912 (re.hypot(im) / span.len() as f64) as f32
1913 };
1914
1915 let retention: Vec<f32> = probes
1916 .iter()
1917 .map(|hz| energy_at(&cleaned, *hz) / energy_at(&noisy, *hz).max(1e-9))
1918 .collect();
1919 let low = retention[0];
1920 for (hz, kept) in probes.iter().zip(retention.iter()) {
1921 assert!(
1922 *kept > 0.5,
1923 "{hz} Hz retained only {kept:.3}; the denoiser is eating the band, not the noise \
1924 (all bands: {retention:?})"
1925 );
1926 assert!(
1927 *kept > low * 0.6,
1928 "{hz} Hz retained {kept:.3} against {low:.3} at 300 Hz — high frequencies are \
1929 being attenuated preferentially, which is how sibilance and speaker identity go \
1930 (all bands: {retention:?})"
1931 );
1932 }
1933 }
1934
1935 #[test]
1939 fn a_clip_shorter_than_its_downsample_ratio_resamples_to_nothing() {
1940 let out = resample_to_speaker_rate(vec![0.25], 192_000);
1941 assert!(
1942 out.is_empty(),
1943 "one sample at 192 kHz is less than half an output sample at \
1944 {SPEAKER_SAMPLE_RATE_HZ} Hz, so it cannot produce one"
1945 );
1946 }
1947
1948 #[test]
1952 fn a_48k_tone_resamples_to_24k_with_its_shape_intact() {
1953 const SOURCE_HZ: u32 = 48_000;
1954 const TONE_HZ: f64 = 1_000.0;
1955 let samples = SOURCE_HZ as usize; let pcm: Vec<f32> = (0..samples)
1957 .map(|n| {
1958 let t = n as f64 / f64::from(SOURCE_HZ);
1959 (std::f64::consts::TAU * TONE_HZ * t).sin() as f32
1960 })
1961 .collect();
1962
1963 let out = resample_to_speaker_rate(pcm, SOURCE_HZ);
1964
1965 let expected_len = SPEAKER_SAMPLE_RATE_HZ as usize;
1966 assert!(
1967 out.len().abs_diff(expected_len) <= 1,
1968 "expected ~{expected_len} samples at {SPEAKER_SAMPLE_RATE_HZ} Hz, got {}",
1969 out.len()
1970 );
1971
1972 let skip = 64;
1975 let interior = out.len() - skip;
1976 let mut worst = 0.0_f32;
1977 for (index, sample) in out.iter().enumerate().take(interior).skip(skip) {
1978 let t = index as f64 / f64::from(SPEAKER_SAMPLE_RATE_HZ);
1979 let ideal = (std::f64::consts::TAU * TONE_HZ * t).sin() as f32;
1980 worst = worst.max((sample - ideal).abs());
1981 }
1982 assert!(
1983 worst < 0.02,
1984 "resampled tone drifted from the analytic reference by {worst}"
1985 );
1986 }
1987
1988 #[test]
1989 fn a_short_speaker_vector_is_refused_rather_than_padded() {
1990 let dir = std::env::temp_dir().join("ftts-synth-tests");
1991 fs::create_dir_all(&dir).expect("temp dir");
1992 let path = dir.join("short.spk");
1993 fs::write(&path, vec![0u8; 64]).expect("write");
1994 let error = read_speaker_vector(&path).expect_err("a short vector must be refused");
1995 let message = error.to_string();
1996 assert!(message.contains("64 bytes"), "{message}");
1997 assert!(message.contains("4096"), "{message}");
1998 }
1999
2000 #[test]
2001 fn a_non_finite_speaker_vector_is_refused() {
2002 let dir = std::env::temp_dir().join("ftts-synth-tests");
2003 fs::create_dir_all(&dir).expect("temp dir");
2004 let path = dir.join("nan.spk");
2005 let mut bytes = vec![0u8; SPEAKER_VECTOR_BYTES];
2006 bytes[0..4].copy_from_slice(&f32::NAN.to_le_bytes());
2007 fs::write(&path, &bytes).expect("write");
2008 let error = read_speaker_vector(&path).expect_err("NaN must be refused");
2009 assert!(error.to_string().contains("index 0"), "{error}");
2010 }
2011
2012 #[test]
2013 fn a_well_formed_speaker_vector_reads_back_exactly() {
2014 let dir = std::env::temp_dir().join("ftts-synth-tests");
2015 fs::create_dir_all(&dir).expect("temp dir");
2016 let path = dir.join("good.spk");
2017 let expected: Vec<f32> = (0..TALKER_HIDDEN).map(|i| i as f32 * 0.001).collect();
2018 let mut bytes = Vec::with_capacity(SPEAKER_VECTOR_BYTES);
2019 for value in &expected {
2020 bytes.extend_from_slice(&value.to_le_bytes());
2021 }
2022 fs::write(&path, &bytes).expect("write");
2023 assert_eq!(read_speaker_vector(&path).expect("read"), expected);
2024 }
2025
2026 #[test]
2027 fn enrollment_writer_refuses_overwrite_and_preserves_the_vector() {
2028 let path = std::env::temp_dir().join(format!(
2029 "ftts-enroll-{}-{}.spk",
2030 std::process::id(),
2031 std::time::SystemTime::now()
2032 .duration_since(std::time::UNIX_EPOCH)
2033 .expect("clock")
2034 .as_nanos()
2035 ));
2036 let expected: Vec<f32> = (0..TALKER_HIDDEN)
2037 .map(|index| index as f32 * 0.125)
2038 .collect();
2039 write_speaker_vector_new(&path, &expected).expect("initial enrollment write");
2040 assert_eq!(
2041 read_speaker_vector(&path).expect("read enrolled vector"),
2042 expected
2043 );
2044 let error = write_speaker_vector_new(&path, &[0.0; TALKER_HIDDEN])
2045 .expect_err("an enrollment must never replace an existing voice");
2046 assert!(error.to_string().contains("without overwriting"), "{error}");
2047 }
2048
2049 #[test]
2050 fn wav_reference_decodes_to_mono_24khz_pcm() {
2051 let path = std::env::temp_dir().join(format!(
2052 "ftts-reference-{}-{}.wav",
2053 std::process::id(),
2054 std::time::SystemTime::now()
2055 .duration_since(std::time::UNIX_EPOCH)
2056 .expect("clock")
2057 .as_nanos()
2058 ));
2059 let pcm: Vec<f32> = (0..1_920)
2060 .map(|index| (index as f32 / 1_920.0 * std::f32::consts::TAU).sin() * 0.25)
2061 .collect();
2062 fs::write(
2063 &path,
2064 ftts_core::audio::encode_wav(&pcm, SPEAKER_SAMPLE_RATE_HZ),
2065 )
2066 .expect("write reference WAV");
2067 let decoded = decode_reference_audio(&path).expect("decode reference WAV");
2068 assert_eq!(decoded.len(), pcm.len());
2069 assert!(decoded.iter().all(|sample| sample.is_finite()));
2070 }
2071
2072 #[test]
2073 fn a_bundle_names_the_file_that_is_actually_missing() {
2074 let dir = std::env::temp_dir().join("ftts-bundle-tests-empty");
2077 fs::create_dir_all(&dir).expect("temp dir");
2078 let error = ModelBundle::resolve(&dir).expect_err("an empty directory is not a bundle");
2079 assert!(error.to_string().contains("model.safetensors"), "{error}");
2080 }
2081
2082 #[test]
2083 fn a_complete_bundle_prefers_its_canonical_artifact_for_synthesis() {
2084 let nonce = std::time::SystemTime::now()
2085 .duration_since(std::time::UNIX_EPOCH)
2086 .expect("clock after epoch")
2087 .as_nanos();
2088 let dir = std::env::temp_dir().join(format!(
2089 "ftts-bundle-canonical-{}-{nonce}",
2090 std::process::id()
2091 ));
2092 fs::create_dir_all(dir.join("speech_tokenizer")).expect("create bundle sidecar directory");
2093 for name in [
2094 CANONICAL_MODEL_BASENAME,
2095 "speech_tokenizer/model.safetensors",
2096 "vocab.json",
2097 "merges.txt",
2098 "tokenizer_config.json",
2099 ] {
2100 fs::write(dir.join(name), []).expect("write bundle fixture sidecar");
2101 }
2102
2103 let expected_artifact = dir.join(CANONICAL_MODEL_BASENAME);
2104 let bundle = ModelBundle::resolve(&dir).expect("complete canonical bundle resolves");
2105 assert_eq!(
2106 bundle.canonical_main.as_deref(),
2107 Some(expected_artifact.as_path())
2108 );
2109 assert!(
2110 !bundle.main.exists(),
2111 "canonical synthesis must not require the raw main checkpoint"
2112 );
2113
2114 let explicit = ModelBundle::resolve(&expected_artifact)
2115 .expect("an explicit canonical artifact resolves against its sidecars");
2116 assert_eq!(
2117 explicit.canonical_main.as_deref(),
2118 Some(expected_artifact.as_path())
2119 );
2120 }
2121}