Skip to main content

ftts_cli/
synth.rs

1//! The `ftts say` synthesis path: text in, 24 kHz PCM out.
2//!
3//! This module is where the CLI stops describing the pipeline and runs it. It resolves a
4//! checkpoint bundle, hydrates the talker and the codec, tokenizes and wraps the text, derives the
5//! prompt header, drives [`TtsEngine::synthesize`] over the real [`QwenGenerator`], and hands the
6//! generated codes to the codec decoder. What comes back is `f32` samples; the WAV writing lives
7//! in `ftts-core::audio` and the sink policy in [`crate::AudioOutput`].
8//!
9//! # Why the text is prepared before the engine runs
10//!
11//! [`TtsEngine::synthesize`] owns text preparation, and normally that is where tokenization
12//! happens. Here it happens once, up front, and the engine is handed a preparer that returns that
13//! exact result. The reason is the cold text embedding: it is `[151936, 2048]`, and materializing
14//! it whole to serve a fifteen-token utterance would cost 1.24 GB. The gather needs the token ids,
15//! the generator needs the gathered table, and the generator must exist before `synthesize` is
16//! called — so the ids have to be known first. The engine still receives, verbatim, the
17//! `PreparedText` a fresh call would have produced; nothing is skipped, only ordered.
18//!
19//! # Speaker conditioning is derived, never invented
20//!
21//! An x-vector prompt conditions on a 1,024-wide speaker embedding. A voice source may be either
22//! a precomputed raw vector (1,024 little-endian `f32`, 4,096 bytes) or reference audio decoded
23//! through the pinned 24 kHz log-mel front end and ECAPA encoder. Neither path accepts a
24//! fabricated vector.
25
26use crate::error::FttsError;
27use ftts_core::{
28    CancellationToken, EngineError, FrameGenerator, GenerationError, NormalizationOptions,
29    NormalizationTrace, PreparedText, SynthesisObserver, SynthesisRequest, TextPreparationError,
30    TextPreparer, TtsEngine,
31};
32use ftts_model_qwen::checkpoint::{
33    CODEC_LANGUAGE_ENGLISH_ID, CheckpointError, CodecCheckpoint, TALKER_HIDDEN, TalkerCheckpoint,
34};
35use ftts_model_qwen::generate::{QwenGenerator, QwenGeneratorConfig};
36use ftts_model_qwen::microdecoder::MicrodecoderConfig;
37use ftts_model_qwen::prompt::{CloneMode, PromptMode};
38use ftts_model_qwen::sampler::SamplingMode;
39use ftts_model_qwen::speaker::{
40    Encoder as SpeakerEncoder, SPEAKER_SAMPLE_RATE_HZ, log_mel_from_24khz_pcm,
41};
42use ftts_model_qwen::talker::TalkerConfig;
43use ftts_model_qwen::tokenizer::{QwenTokenizer, TokenizerFiles};
44use std::fs;
45use std::fs::OpenOptions;
46use std::path::{Path, PathBuf};
47use symphonia::core::audio::SampleBuffer;
48use symphonia::core::codecs::DecoderOptions;
49use symphonia::core::errors::Error as SymphoniaError;
50use symphonia::core::formats::FormatOptions;
51use symphonia::core::io::MediaSourceStream;
52use symphonia::core::meta::MetadataOptions;
53use symphonia::core::probe::Hint;
54use symphonia::default::{get_codecs, get_probe};
55
56/// Bytes in a speaker-vector file: 1,024 little-endian `f32`.
57pub const SPEAKER_VECTOR_BYTES: usize = TALKER_HIDDEN * 4;
58
59const CANONICAL_MODEL_BASENAME: &str = "qwen3-tts-12hz-0.6b-base.fttsq";
60
61fn checkpoint_error(error: CheckpointError) -> FttsError {
62    FttsError::ArtifactFormat(error.to_string())
63}
64
65/// The model resources `ftts say` needs, located relative to one model path.
66#[derive(Clone, Debug)]
67pub struct ModelBundle {
68    /// Directory holding the artifact sidecars and tokenizer files.
69    pub root: PathBuf,
70    /// The raw main checkpoint, retained for enrollment-only components that have not yet gained
71    /// a canonical-artifact accessor.
72    pub main: PathBuf,
73    /// The portable main-weight artifact selected for synthesis, when present.
74    pub canonical_main: Option<PathBuf>,
75    /// The codec decoder checkpoint.
76    pub codec: PathBuf,
77}
78
79impl ModelBundle {
80    /// Resolve a bundle from `--model`, which may name the directory, a `.fttsq`, or
81    /// `model.safetensors`.
82    ///
83    /// # Errors
84    ///
85    /// [`FttsError::ModelNotFound`] naming the exact missing file, so a partial download is
86    /// diagnosable without guessing which of the four is absent.
87    pub fn resolve(model: &Path) -> Result<Self, FttsError> {
88        let root = if model.is_dir() {
89            model.to_path_buf()
90        } else {
91            model
92                .parent()
93                .ok_or_else(|| {
94                    FttsError::ModelNotFound(format!(
95                        "model path {} has no parent directory",
96                        model.display()
97                    ))
98                })?
99                .to_path_buf()
100        };
101        let canonical_main = if model.is_dir() {
102            let canonical = root.join(CANONICAL_MODEL_BASENAME);
103            if canonical.is_file() {
104                Some(canonical)
105            } else {
106                None
107            }
108        } else if model.extension().and_then(|extension| extension.to_str()) == Some("fttsq") {
109            Some(model.to_path_buf())
110        } else {
111            None
112        };
113        let main = root.join("model.safetensors");
114        let codec = root.join("speech_tokenizer/model.safetensors");
115        let (main_label, main_path) = match canonical_main.as_ref() {
116            Some(path) => ("canonical talker artifact", path),
117            None => ("talker checkpoint", &main),
118        };
119        for (label, path) in [
120            (main_label, main_path),
121            ("codec checkpoint", &codec),
122            ("tokenizer vocabulary", &root.join("vocab.json")),
123            ("tokenizer merges", &root.join("merges.txt")),
124            ("tokenizer config", &root.join("tokenizer_config.json")),
125        ] {
126            if !path.is_file() {
127                return Err(FttsError::ModelNotFound(format!(
128                    "{label} is missing at {}; `ftts say` needs a complete model directory \
129                     ({CANONICAL_MODEL_BASENAME} or model.safetensors, \
130                     speech_tokenizer/model.safetensors, \
131                     vocab.json, merges.txt, tokenizer_config.json)",
132                    path.display()
133                )));
134            }
135        }
136        Ok(Self {
137            root,
138            main,
139            canonical_main,
140            codec,
141        })
142    }
143}
144
145/// Every weight and table one `say` needs, hydrated once.
146pub struct LoadedModel {
147    talker: TalkerCheckpoint,
148    codec: CodecCheckpoint,
149    tokenizer: QwenTokenizer,
150    /// The checkpoint's own digest-verified mapping of the canonical artifact, shared so the
151    /// int8 route hydrates its Q8 tables from it (proven byte-identical to requantizing the
152    /// widened f32 copies, scales included). Never re-opened: every `MappedFttsq::open`
153    /// re-verifies the whole artifact's digests, ~1.3 GB of hashing.
154    artifact: Option<std::sync::Arc<ftts_artifacts::fttsq::MappedFttsq>>,
155}
156
157impl LoadedModel {
158    /// Hydrate the bundle. This reads gigabytes and is the slow step of a cold run.
159    ///
160    /// # Errors
161    ///
162    /// If any checkpoint or tokenizer file is unreadable or not the pinned model.
163    pub fn load(bundle: &ModelBundle) -> Result<Self, FttsError> {
164        let read = |name: &str| -> Result<String, FttsError> {
165            let path = bundle.root.join(name);
166            fs::read_to_string(&path).map_err(|error| {
167                FttsError::ArtifactFormat(format!("cannot read {}: {error}", path.display()))
168            })
169        };
170        let vocab = read("vocab.json")?;
171        let merges = read("merges.txt")?;
172        let config = read("tokenizer_config.json")?;
173
174        // The three heavyweight hydrations are independent, so the codec checkpoint and the
175        // tokenizer build overlap the talker load instead of queueing behind it. Each result is
176        // computed exactly as it was serially; only wall time changes.
177        let (talker, codec, tokenizer) = std::thread::scope(|scope| {
178            let codec = scope.spawn(|| CodecCheckpoint::load(&bundle.codec));
179            let tokenizer = scope.spawn(|| {
180                QwenTokenizer::from_files_using_environment(TokenizerFiles {
181                    vocab_json: &vocab,
182                    merges_txt: &merges,
183                    tokenizer_config_json: &config,
184                })
185            });
186            let talker = match bundle.canonical_main.as_deref() {
187                // The elision mirrors the generator's own hydration decision for this process:
188                // stacks that will run artifact-native int8 skip their dead f32 projections
189                // (~2.5 GB of widening nobody reads).
190                Some(path) => TalkerCheckpoint::load_fttsq_elided(
191                    path,
192                    ftts_model_qwen::generate::hot_elision_from_environment(),
193                ),
194                None => TalkerCheckpoint::load(&bundle.main),
195            };
196            (
197                talker,
198                codec.join().expect("codec loader panicked"),
199                tokenizer.join().expect("tokenizer builder panicked"),
200            )
201        });
202        let tokenizer = tokenizer
203            .map_err(|error| FttsError::ArtifactFormat(format!("tokenizer unusable: {error}")))?;
204
205        let talker = talker.map_err(checkpoint_error)?;
206        // Shared, not re-opened: a second MappedFttsq::open would re-verify the whole artifact's
207        // digests (~1.3 GB of hashing) for a mapping the checkpoint already carries.
208        let artifact = talker.artifact().cloned();
209        Ok(Self {
210            talker,
211            codec: codec.map_err(checkpoint_error)?,
212            tokenizer,
213            artifact,
214        })
215    }
216}
217
218/// Read a precomputed 1,024-wide speaker vector.
219///
220/// See the module docs on why this is a raw vector rather than a `.ftvoice` pack.
221///
222/// # Errors
223///
224/// [`FttsError::Input`] when the file is unreadable or is not exactly
225/// [`SPEAKER_VECTOR_BYTES`] bytes — a truncated vector would otherwise be padded with silence and
226/// change the voice in a way only listening could detect.
227pub fn read_speaker_vector(path: &Path) -> Result<Vec<f32>, FttsError> {
228    let bytes = fs::read(path).map_err(|error| {
229        FttsError::Input(format!(
230            "cannot read speaker vector {}: {error}",
231            path.display()
232        ))
233    })?;
234    if bytes.len() != SPEAKER_VECTOR_BYTES {
235        return Err(FttsError::Input(format!(
236            "speaker vector {} is {} bytes; `ftts say --voice` expects exactly {} \
237             ({TALKER_HIDDEN} little-endian f32)",
238            path.display(),
239            bytes.len(),
240            SPEAKER_VECTOR_BYTES
241        )));
242    }
243    let vector: Vec<f32> = bytes
244        .as_chunks::<4>()
245        .0
246        .iter()
247        .map(|quad| f32::from_le_bytes(*quad))
248        .collect();
249    if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
250        return Err(FttsError::Input(format!(
251            "speaker vector {} holds a non-finite value at index {index}; it would poison every \
252             prefill position it is summed into",
253            path.display()
254        )));
255    }
256    Ok(vector)
257}
258
259/// Derive an x-vector from an enrolled raw vector or a real reference recording.
260/// What a `--denoise` enrollment measured, so the CLI can report the effect rather than assert it.
261#[derive(Clone, Copy, Debug)]
262pub struct DenoiseReport {
263    /// Pause floor of the decoded reference, before denoising.
264    pub before_dbfs: f32,
265    /// Pause floor after denoising.
266    pub after_dbfs: f32,
267}
268
269/// Which reference-cleanup stages to run, and where each reports what it measured.
270///
271/// Both default to off. Every stage changes the enrolled identity, so none of them is applied on
272/// a user's behalf — a lever that can alter who the clone sounds like is opted into by name.
273#[derive(Default)]
274pub struct ReferenceCleanup<'a> {
275    /// Spectral-subtract the stationary noise floor.
276    pub denoise: Option<&'a mut Option<DenoiseReport>>,
277    /// Remove late reverberation.
278    pub dereverb: Option<&'a mut Option<DereverbReport>>,
279}
280
281/// Derive a speaker vector from a voice source: a raw x-vector file, or reference audio.
282///
283/// Passing `Some` for a cleanup slot opts the reference into that stage and fills the slot with
284/// what it measured. Dereverberation runs first: it is a linear operation on the observed signal,
285/// so applying it before the noise floor is estimated keeps that estimate from being fitted to a
286/// signal the next stage is about to change.
287///
288/// # Errors
289///
290/// When the source cannot be read, decoded, resampled, or encoded into a finite x-vector.
291pub fn speaker_from_voice(
292    bundle: &ModelBundle,
293    path: &Path,
294    cleanup: ReferenceCleanup<'_>,
295) -> Result<Vec<f32>, FttsError> {
296    let bytes = fs::read(path).map_err(|error| {
297        FttsError::Input(format!(
298            "cannot read voice source {}: {error}",
299            path.display()
300        ))
301    })?;
302    if bytes.len() == SPEAKER_VECTOR_BYTES {
303        return decode_speaker_vector(path, &bytes);
304    }
305    let pcm = decode_reference_audio_any(path)?;
306    let ReferenceCleanup { denoise, dereverb } = cleanup;
307    let pcm = match dereverb {
308        Some(report) => {
309            let before = reverb_time_s(&pcm);
310            let dried = dereverb_reference(&pcm);
311            let after = reverb_time_s(&dried);
312            if let (Some(before), Some(after)) = (before, after) {
313                *report = Some(DereverbReport {
314                    before_rt60_s: before,
315                    after_rt60_s: after,
316                });
317            }
318            dried
319        }
320        None => pcm,
321    };
322    let pcm = match denoise {
323        Some(report) => {
324            let before = pause_floor_dbfs(&pcm);
325            let cleaned = denoise_reference(&pcm);
326            *report = Some(DenoiseReport {
327                before_dbfs: before,
328                after_dbfs: pause_floor_dbfs(&cleaned),
329            });
330            cleaned
331        }
332        None => pcm,
333    };
334    let mel = log_mel_from_24khz_pcm(&pcm)
335        .map_err(|error| FttsError::Input(format!("cannot extract speaker features: {error}")))?;
336    let encoder = match bundle.canonical_main.as_deref() {
337        Some(artifact) => SpeakerEncoder::load_fttsq(artifact),
338        None => SpeakerEncoder::load(&bundle.main),
339    }
340    .map_err(checkpoint_error)?;
341    let vector = encoder.encode(&mel.values, mel.frames);
342    if vector.iter().all(|value| value.is_finite()) {
343        Ok(vector)
344    } else {
345        Err(FttsError::Input(
346            "speaker encoder produced a non-finite x-vector; refusing to condition synthesis"
347                .to_owned(),
348        ))
349    }
350}
351
352/// Write a raw x-vector without replacing an existing enrollment result.
353pub fn write_speaker_vector_new(path: &Path, vector: &[f32]) -> Result<(), FttsError> {
354    if vector.len() != TALKER_HIDDEN {
355        return Err(FttsError::Input(format!(
356            "cannot write {}-wide speaker vector; expected {TALKER_HIDDEN}",
357            vector.len()
358        )));
359    }
360    if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
361        return Err(FttsError::Input(format!(
362            "cannot write speaker vector with a non-finite value at index {index}"
363        )));
364    }
365    let mut bytes = Vec::with_capacity(SPEAKER_VECTOR_BYTES);
366    for value in vector {
367        bytes.extend_from_slice(&value.to_le_bytes());
368    }
369    use std::io::Write;
370    let mut file = OpenOptions::new()
371        .write(true)
372        .create_new(true)
373        .open(path)
374        .map_err(|error| {
375            FttsError::Input(format!(
376                "cannot create enrolled voice {} without overwriting an existing file: {error}",
377                path.display()
378            ))
379        })?;
380    file.write_all(&bytes).map_err(|error| {
381        FttsError::Input(format!(
382            "cannot write enrolled voice {}: {error}",
383            path.display()
384        ))
385    })
386}
387
388/// Replaces an existing enrolled voice, keeping the displaced one alongside it.
389///
390/// Enrollment is cheap to redo but a voice is not always cheap to re-record, and the reference a
391/// `.spk` came from may be long gone. The previous vector is copied to `<path>.bak` before the new
392/// one lands, so a mistaken overwrite is one `mv` away from undone rather than unrecoverable.
393///
394/// # Errors
395///
396/// When the vector is malformed, the backup cannot be written, or the file cannot be replaced.
397pub fn replace_speaker_vector(path: &Path, vector: &[f32]) -> Result<PathBuf, FttsError> {
398    let backup = path.with_extension("spk.bak");
399    fs::copy(path, &backup).map_err(|error| {
400        FttsError::Input(format!(
401            "cannot back up the existing voice {} to {}: {error}",
402            path.display(),
403            backup.display()
404        ))
405    })?;
406    // Write the replacement to a sibling first, then rename over the target: a crash mid-write
407    // must not leave a half-written vector where a valid voice used to be.
408    let staging = path.with_extension("spk.incoming");
409    if staging.exists() {
410        fs::remove_file(&staging).map_err(|error| {
411            FttsError::Input(format!(
412                "cannot clear the stale staging file {}: {error}",
413                staging.display()
414            ))
415        })?;
416    }
417    write_speaker_vector_new(&staging, vector)?;
418    fs::rename(&staging, path).map_err(|error| {
419        FttsError::Input(format!(
420            "cannot replace {} with the new voice: {error}",
421            path.display()
422        ))
423    })?;
424    Ok(backup)
425}
426
427fn decode_speaker_vector(path: &Path, bytes: &[u8]) -> Result<Vec<f32>, FttsError> {
428    let vector: Vec<f32> = bytes
429        .as_chunks::<4>()
430        .0
431        .iter()
432        .map(|quad| f32::from_le_bytes(*quad))
433        .collect();
434    if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
435        return Err(FttsError::Input(format!(
436            "speaker vector {} holds a non-finite value at index {index}; it would poison every \
437             prefill position it is summed into",
438            path.display()
439        )));
440    }
441    Ok(vector)
442}
443
444/// Container formats the embedded decoder does not read; these route through a system decoder,
445/// mirroring how output encoding shells out — synthesis and enrollment themselves never depend
446/// on one.
447const SYSTEM_DECODED_EXTENSIONS: [&str; 6] = ["m4a", "mp3", "aac", "mp4", "ogg", "opus"];
448
449/// Decodes reference audio of any supported container to mono f32 PCM.
450///
451/// WAV and FLAC decode through the embedded pure-Rust path. Compressed containers (m4a, mp3, …)
452/// are first transcoded to a temporary WAV by the first system decoder found — `afconvert` on
453/// macOS, then `ffmpeg` — with a clear error naming both tools when neither exists.
454fn decode_reference_audio_any(path: &Path) -> Result<Vec<f32>, FttsError> {
455    let extension = path
456        .extension()
457        .and_then(|extension| extension.to_str())
458        .map(str::to_ascii_lowercase);
459    let needs_system_decoder = extension
460        .as_deref()
461        .is_some_and(|extension| SYSTEM_DECODED_EXTENSIONS.contains(&extension));
462    if !needs_system_decoder {
463        return decode_reference_audio(path);
464    }
465
466    let staging = std::env::temp_dir().join(format!(
467        "ftts-enroll-{}-{}.wav",
468        std::process::id(),
469        path.file_stem()
470            .and_then(|stem| stem.to_str())
471            .unwrap_or("reference")
472    ));
473    let attempts: &[(&str, Vec<&std::ffi::OsStr>)] = &[
474        // Both decoders are told to resample to the speaker encoder's pinned 24 kHz mono here
475        // rather than leaving the source rate intact: phone and Mac voice memos default to
476        // 44.1/48 kHz, and a transcode that preserves them would only move the failure to the
477        // enrollment rate check (frankentts-gra).
478        (
479            "afconvert",
480            vec![
481                "-f".as_ref(),
482                "WAVE".as_ref(),
483                "-d".as_ref(),
484                "LEI16@24000".as_ref(),
485                "-c".as_ref(),
486                "1".as_ref(),
487                path.as_os_str(),
488                staging.as_os_str(),
489            ],
490        ),
491        (
492            "ffmpeg",
493            vec![
494                "-y".as_ref(),
495                "-loglevel".as_ref(),
496                "error".as_ref(),
497                "-i".as_ref(),
498                path.as_os_str(),
499                "-acodec".as_ref(),
500                "pcm_s16le".as_ref(),
501                "-ar".as_ref(),
502                "24000".as_ref(),
503                "-ac".as_ref(),
504                "1".as_ref(),
505                staging.as_os_str(),
506            ],
507        ),
508    ];
509    let mut ran = false;
510    for (tool, arguments) in attempts {
511        match std::process::Command::new(tool).args(arguments).status() {
512            Ok(status) if status.success() => {
513                ran = true;
514                break;
515            }
516            Ok(status) => {
517                let _ = fs::remove_file(&staging);
518                return Err(FttsError::Input(format!(
519                    "{tool} failed decoding reference audio {} (exit {status})",
520                    path.display()
521                )));
522            }
523            Err(_) => continue, // tool not installed; try the next one
524        }
525    }
526    if !ran {
527        return Err(FttsError::Input(format!(
528            "reference audio {} is a compressed container and no system decoder was found; \
529             install afconvert (macOS) or ffmpeg, or supply WAV/FLAC",
530            path.display()
531        )));
532    }
533    let decoded = decode_reference_audio(&staging);
534    let _ = fs::remove_file(&staging);
535    decoded
536}
537
538fn decode_reference_audio(path: &Path) -> Result<Vec<f32>, FttsError> {
539    let file = fs::File::open(path).map_err(|error| {
540        FttsError::Input(format!(
541            "cannot open reference audio {}: {error}",
542            path.display()
543        ))
544    })?;
545    let mut hint = Hint::new();
546    if let Some(extension) = path.extension().and_then(|extension| extension.to_str()) {
547        hint.with_extension(extension);
548    }
549    let stream = MediaSourceStream::new(Box::new(file), Default::default());
550    let probed = get_probe()
551        .format(
552            &hint,
553            stream,
554            &FormatOptions::default(),
555            &MetadataOptions::default(),
556        )
557        .map_err(|error| {
558            FttsError::Input(format!(
559                "cannot identify reference audio {}: {error}",
560                path.display()
561            ))
562        })?;
563    let mut format = probed.format;
564    let track = format.default_track().ok_or_else(|| {
565        FttsError::Input(format!(
566            "reference audio {} has no default audio track",
567            path.display()
568        ))
569    })?;
570    let track_id = track.id;
571    let mut decoder = get_codecs()
572        .make(&track.codec_params, &DecoderOptions::default())
573        .map_err(|error| {
574            FttsError::Input(format!(
575                "cannot decode reference audio {}: {error}",
576                path.display()
577            ))
578        })?;
579    let mut sample_rate = None;
580    let mut mono = Vec::new();
581    loop {
582        let packet = match format.next_packet() {
583            Ok(packet) => packet,
584            Err(SymphoniaError::IoError(error))
585                if error.kind() == std::io::ErrorKind::UnexpectedEof =>
586            {
587                break;
588            }
589            Err(error) => {
590                return Err(FttsError::Input(format!(
591                    "cannot read reference audio {}: {error}",
592                    path.display()
593                )));
594            }
595        };
596        if packet.track_id() != track_id {
597            continue;
598        }
599        let decoded = decoder.decode(&packet).map_err(|error| {
600            FttsError::Input(format!(
601                "cannot decode reference audio {}: {error}",
602                path.display()
603            ))
604        })?;
605        let spec = *decoded.spec();
606        match sample_rate {
607            Some(rate) if rate != spec.rate => {
608                return Err(FttsError::Input(format!(
609                    "reference audio {} changed sample rate mid-stream ({rate} to {} Hz)",
610                    path.display(),
611                    spec.rate
612                )));
613            }
614            None => sample_rate = Some(spec.rate),
615            Some(_) => {}
616        }
617        let channels = spec.channels.count();
618        let mut samples = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
619        samples.copy_interleaved_ref(decoded);
620        for frame in samples.samples().chunks_exact(channels) {
621            mono.push(frame.iter().sum::<f32>() / channels as f32);
622        }
623    }
624    let rate = sample_rate.ok_or_else(|| {
625        FttsError::Input(format!(
626            "reference audio {} contains no decodable samples",
627            path.display()
628        ))
629    })?;
630    if mono.is_empty() {
631        return Err(FttsError::Input(format!(
632            "reference audio {} contains no PCM samples",
633            path.display()
634        )));
635    }
636    let pcm = resample_to_speaker_rate(mono, rate);
637    // Downsampling shortens the signal, and a clip of a few samples at a high source rate can
638    // round to nothing. The mel front end would then see an empty slice, so the emptiness check
639    // has to be made against the PCM actually handed on, not only against what was decoded.
640    if pcm.is_empty() {
641        return Err(FttsError::Input(format!(
642            "reference audio {} is too short to resample from {rate} Hz to \
643             {SPEAKER_SAMPLE_RATE_HZ} Hz; supply a longer recording",
644            path.display()
645        )));
646    }
647    Ok(pcm)
648}
649
650/// Resamples decoded mono PCM to the speaker encoder's pinned rate.
651///
652/// Compressed references already arrive at 24 kHz because the system decoder is told to convert
653/// (`frankentts-gra`), but a `.wav` or `.flac` is read directly and can be any rate — 44.1 and 48
654/// kHz being what every phone, Mac voice memo, and DAW export actually produces. Refusing those
655/// pushed the identical resample onto the user as an `ffmpeg` incantation, so it happens here.
656///
657/// Audio already at the pinned rate is returned untouched, so this cannot perturb any existing
658/// enrollment: it only turns a former hard error into a working path.
659///
660/// Windowed-sinc (Lanczos-3) with the kernel cutoff clamped to the lower of the two rates, which
661/// is what suppresses aliasing on the common downsampling direction. Taps are normalized by their
662/// own sum so DC gain stays 1 even where the window runs off the ends of the signal.
663fn resample_to_speaker_rate(mono: Vec<f32>, from_rate: u32) -> Vec<f32> {
664    if from_rate == SPEAKER_SAMPLE_RATE_HZ {
665        return mono;
666    }
667    // Six lobes, not three: with the cutoff at the output Nyquist a 3-lobe kernel's transition
668    // band sits inside the passband — measured -2.2 dB at 10 kHz for 48->24 kHz and only ~18 dB
669    // of alias rejection, right where the speaker encoder reads sibilance. Six lobes halves the
670    // transition width and pushes rejection past 40 dB for double the (still trivial) tap count.
671    const LOBES: f64 = 6.0;
672    let ratio = f64::from(SPEAKER_SAMPLE_RATE_HZ) / f64::from(from_rate);
673    let cutoff = ratio.min(1.0);
674    let half = (LOBES / cutoff).ceil() as isize;
675    let out_len = ((mono.len() as f64) * ratio).round() as usize;
676
677    let mut out = Vec::with_capacity(out_len);
678    for index in 0..out_len {
679        let center = index as f64 / ratio;
680        let first = center.floor() as isize - half + 1;
681        let mut acc = 0.0_f64;
682        let mut norm = 0.0_f64;
683        for tap in first..first + 2 * half {
684            if tap < 0 {
685                continue;
686            }
687            let Some(sample) = mono.get(tap as usize) else {
688                break;
689            };
690            let weight = lanczos_tap(center - tap as f64, cutoff, LOBES);
691            acc += weight * f64::from(*sample);
692            norm += weight;
693        }
694        out.push(if norm.abs() > 1e-12 {
695            (acc / norm) as f32
696        } else {
697            0.0
698        });
699    }
700    out
701}
702
703/// STFT window for reference denoising: 512 samples is ~21 ms at the pinned 24 kHz, long enough
704/// to resolve a noise floor between words and short enough not to smear plosives.
705const DENOISE_FRAME: usize = 512;
706
707/// Three-quarter overlap. Hann at hop `N/4` overlap-adds smoothly, which is what keeps gain
708/// changes from becoming audible frame edges.
709const DENOISE_HOP: usize = DENOISE_FRAME / 4;
710
711/// Decision-directed smoothing for the a priori SNR. Ephraim and Malah's 0.98 is calmer but lags
712/// onsets; 0.92 tracks a voice that starts and stops mid-recording.
713const DD_ALPHA: f32 = 0.92;
714
715/// Floor gain, −35 dB. OM-LSA never gates a bin fully closed: leaving a quiet, *stationary* bed
716/// is what stops residual noise from flickering into musical tones.
717///
718/// This is the right knob for "deeper pauses", and the only safe one — it applies where the
719/// presence probability has already decided a bin is noise, so lowering it buys silence between
720/// words without touching a bin the estimator thinks holds voice. Reaching for a more aggressive
721/// *noise estimate* instead is what damages speech.
722const OMLSA_GAIN_FLOOR: f32 = 0.017_782_79;
723
724/// Frames per block over which each bin's noise floor is estimated. ~1.4 s at this hop: long
725/// enough that speech is sparse within a block, short enough to follow room tone that drifts.
726const NOISE_BLOCK_FRAMES: usize = 256;
727
728/// Prior probability that a bin holds no speech, used in the likelihood ratio. Slightly above a
729/// half so that ambiguous bins lean toward suppression rather than passing noise through.
730const SPEECH_ABSENCE_PRIOR: f32 = 0.6;
731
732/// Bias compensation for reading a low quantile of an exponentially distributed power as its
733/// mean: for `Exp(mu)` the q-quantile is `-mu ln(1-q)`, so the 10th percentile UNDERSTATES the
734/// mean 9.49x. Without this factor, pure-pause frames measure a posteriori SNRs of ~7-10 against
735/// the uncorrected floor, the presence estimator reads them as speech, and the gain never
736/// approaches the floor (measured: 2.5 dB of pause reduction instead of ~20). This is the same
737/// role IMCRA's B_min plays for its minimum statistic, recomputed for the quantile used here.
738#[allow(clippy::excessive_precision)]
739const NOISE_QUANTILE_BIAS: f32 = 9.491_221; // 1 / -ln(1 - NOISE_INIT_QUANTILE)
740
741/// Quantile of each bin's power, across the whole recording, taken as its initial noise floor.
742/// Speech is sparse in time, so a low quantile of a bin is the room rather than the voice.
743const NOISE_INIT_QUANTILE: f32 = 0.1;
744
745/// Single-channel speech enhancement: MMSE-LSA gains, decision-directed SNR, OM-LSA presence
746/// weighting, over a noise floor initialised offline and then tracked recursively.
747///
748/// Enrollment noise is not cosmetic: it is encoded into the x-vector and then reproduced in every
749/// utterance the cloned voice speaks (measured — cleaning a real 53 s reference dropped the
750/// synthesized output's pause floor by 19.5 dB). This removes the stationary part of it.
751///
752/// **Why this and not spectral subtraction.** Subtracting an estimated noise magnitude minimises
753/// squared error in the *spectrum*, which is the wrong objective for something a listener judges
754/// and a speaker encoder reads: it punches holes in low-SNR bins, producing musical noise, and it
755/// removes real signal along with the noise (measured here at 33% of peak burst energy before
756/// this replaced it). Three pieces fix that, and they compose:
757///
758/// 1. **MMSE-LSA** (Ephraim & Malah 1985) estimates the *log* amplitude, matching how loudness is
759///    perceived, and yields the gain `ξ/(1+ξ) · exp(½·E₁(ν))`. The exponential-integral term is
760///    what makes it gentle where the a posteriori SNR is uncertain instead of gating hard.
761/// 2. **Decision-directed a priori SNR** (same paper) smooths ξ across frames using the previous
762///    frame's own estimate. This is the specific mechanism that suppresses musical noise: isolated
763///    noise peaks never get a confident ξ, so they are never sharply attenuated *or* passed.
764/// 3. **OM-LSA** (Cohen & Berdugo 2001) blends that gain toward the floor by the speech-presence
765///    probability, `G = G_LSA^p · G_min^(1−p)`, so bins that are probably noise settle to a
766///    constant bed rather than being tracked.
767///
768/// **Why the noise floor is initialised offline rather than by minimum statistics.** IMCRA's
769/// online minimum tracking exists because a streaming denoiser cannot see the future. Enrollment
770/// can: the file is already on disk, so a low quantile of each bin over the whole recording is a
771/// better starting floor than any causal estimator's, with none of the convergence transient.
772/// An earlier revision here did run minimum statistics, and it is instructive why that was
773/// removed rather than debugged: seeded from frame 0 of a reference that opens on speech, the
774/// refined minimum locked above the speech level, which drove the presence probability to zero,
775/// which unfroze the noise update, which let the noise estimate absorb the voice — a positive
776/// feedback that left ~10% of every burst after the first. Speech presence here instead comes
777/// from the likelihood ratio in ξ and ν, which is self-correcting: it cannot conclude "no speech"
778/// about a bin whose own a priori SNR is high.
779///
780/// Nonstationary noise is still tracked, by the recursive average that the presence probability
781/// gates — the offline quantile only sets where that average starts.
782///
783/// This is the state of the art among methods that need no trained weights. Neural enhancers
784/// (DeepFilterNet and friends) do beat it, at the cost of shipping and running another model —
785/// which is not a trade this CLI should make silently for an enrollment preprocessing step.
786///
787/// Deliberately conservative even so: the speaker encoder reads breath, sibilance, and room as
788/// part of identity, so this stays opt-in (`--denoise`) per the project's doctrine that a lever
789/// which can damage speaker identity ships behind a named switch until blind listening clears it.
790///
791/// Phase is preserved untouched; only per-bin magnitude is scaled.
792fn denoise_reference(pcm: &[f32]) -> Vec<f32> {
793    // A floor estimated from a handful of frames is just the clip's own spectrum; below ~a
794    // quarter second there is nothing honest to subtract, so the clip passes through untouched
795    // (a single-frame "estimate" measured as flattening the whole clip toward the gain floor).
796    const DENOISE_MIN_FRAMES: usize = 32;
797    if pcm.len() < DENOISE_FRAME + (DENOISE_MIN_FRAMES - 1) * DENOISE_HOP {
798        return pcm.to_vec();
799    }
800    let mut planner = rustfft::FftPlanner::<f32>::new();
801    let forward = planner.plan_fft_forward(DENOISE_FRAME);
802    let inverse = planner.plan_fft_inverse(DENOISE_FRAME);
803
804    let window: Vec<f32> = (0..DENOISE_FRAME)
805        .map(|n| {
806            let phase = std::f32::consts::TAU * n as f32 / DENOISE_FRAME as f32;
807            0.5 - 0.5 * phase.cos()
808        })
809        .collect();
810
811    let bins = DENOISE_FRAME / 2 + 1;
812    let starts: Vec<usize> = (0..=pcm.len() - DENOISE_FRAME)
813        .step_by(DENOISE_HOP)
814        .collect();
815
816    // Pass 1: every frame's power spectrum. Only the magnitudes are kept; retaining the complex
817    // frames would save the second FFT at 8× the memory, which a several-minute reference feels.
818    let mut powers: Vec<Vec<f32>> = Vec::with_capacity(starts.len());
819    let mut scratch: Vec<rustfft::num_complex::Complex<f32>> =
820        vec![rustfft::num_complex::Complex::new(0.0, 0.0); DENOISE_FRAME];
821    for &start in &starts {
822        for (slot, n) in scratch.iter_mut().zip(0..DENOISE_FRAME) {
823            *slot = rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0);
824        }
825        forward.process(&mut scratch);
826        powers.push((0..bins).map(|bin| scratch[bin].norm_sqr()).collect());
827    }
828
829    // Noise floor per bin: the MINIMUM over blocks of a low within-block quantile.
830    //
831    // There is deliberately no feedback here. A recursive noise average has to be gated by a
832    // speech-presence estimate, which in turn divides by the noise — and any error in that loop
833    // compounds: one speech frame admitted into the floor raises it, which lowers the presence
834    // estimate, which admits more speech. Both earlier revisions of this function died that way.
835    // Reading the whole file at once removes the loop rather than tuning it.
836    //
837    // The min-over-blocks reduction is load-bearing: a bare per-block quantile assumes speech is
838    // sparse WITHIN every 1.4 s block, and a bin that stays voiced across one whole block (a held
839    // vowel, a low harmonic mid-sentence) would have its own signal adopted as that block's floor
840    // and be gated to the floor gain — measured at ~28 dB of deletion on a sustained tone. Taking
841    // the minimum across blocks only requires the bin to be quiet somewhere in the recording,
842    // which is what "noise floor" actually means.
843    let blocks = powers.len().div_ceil(NOISE_BLOCK_FRAMES);
844    let mut noise = vec![f32::INFINITY; bins];
845    let mut column: Vec<f32> = Vec::with_capacity(NOISE_BLOCK_FRAMES);
846    for block in 0..blocks {
847        let span = block * NOISE_BLOCK_FRAMES..((block + 1) * NOISE_BLOCK_FRAMES).min(powers.len());
848        for (bin, slot) in noise.iter_mut().enumerate() {
849            column.clear();
850            column.extend(powers[span.clone()].iter().map(|frame| frame[bin]));
851            column.sort_by(f32::total_cmp);
852            let rank = ((column.len() as f32 - 1.0) * NOISE_INIT_QUANTILE).round() as usize;
853            *slot = slot.min(column[rank].max(1e-12) * NOISE_QUANTILE_BIAS);
854        }
855    }
856
857    // Decision-directed state: last frame's gain and a posteriori SNR, per bin.
858    let mut prev_gain = vec![1.0_f32; bins];
859    let mut prev_gamma = vec![1.0_f32; bins];
860
861    let mut out = vec![0.0_f32; pcm.len()];
862    let mut weight = vec![0.0_f32; pcm.len()];
863
864    for (index, &start) in starts.iter().enumerate() {
865        let mut frame: Vec<rustfft::num_complex::Complex<f32>> = (0..DENOISE_FRAME)
866            .map(|n| rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0))
867            .collect();
868        forward.process(&mut frame);
869        let power = &powers[index];
870
871        for bin in 0..bins {
872            let gamma = (power[bin] / noise[bin]).min(1e6);
873            let xi = (DD_ALPHA * prev_gain[bin].powi(2) * prev_gamma[bin]
874                + (1.0 - DD_ALPHA) * (gamma - 1.0).max(0.0))
875            .max(1e-6);
876
877            let nu = (xi / (1.0 + xi)) * gamma;
878            let lsa =
879                ((xi / (1.0 + xi)) * (0.5 * exponential_integral_e1(nu)).exp()).clamp(0.0, 1.0);
880
881            // Speech-presence probability by the likelihood ratio (Ephraim & Malah's signal
882            // presence uncertainty). High ξ with a matching ν drives this to 1, so a bin that is
883            // plainly speech can never be talked into being noise.
884            let odds = SPEECH_ABSENCE_PRIOR / (1.0 - SPEECH_ABSENCE_PRIOR);
885            let presence = 1.0 / (1.0 + odds * (1.0 + xi) * (-nu).exp());
886            let presence = presence.clamp(0.0, 1.0);
887
888            let gain = (lsa.max(OMLSA_GAIN_FLOOR).powf(presence)
889                * OMLSA_GAIN_FLOOR.powf(1.0 - presence))
890            .clamp(OMLSA_GAIN_FLOOR, 1.0);
891
892            prev_gain[bin] = gain;
893            prev_gamma[bin] = gamma;
894
895            frame[bin] *= gain;
896            let mirror = DENOISE_FRAME - bin;
897            // DC (bin 0) has no mirror, and Nyquist (bin N/2) *is* its own mirror — scaling it
898            // through this branch as well would apply `gain` twice there.
899            if mirror != bin && mirror < DENOISE_FRAME {
900                frame[mirror] *= gain;
901            }
902        }
903
904        inverse.process(&mut frame);
905        let scale = 1.0 / DENOISE_FRAME as f32;
906        for n in 0..DENOISE_FRAME {
907            out[start + n] += frame[n].re * scale * window[n];
908            weight[start + n] += window[n] * window[n];
909        }
910    }
911
912    // A sample under-covered by the window stack cannot be normalized honestly: near the edges
913    // out[n] is dominated by circular-convolution leakage from the rest of the frame, and
914    // dividing that by a window energy as small as ~2e-6 manufactures a spike (measured 1.5x
915    // input peak with a bare non-zero guard). Anything below a tenth of the steady-state COLA
916    // sum (1.5 for periodic Hann at hop N/4) keeps the original PCM instead.
917    const WOLA_MIN_WEIGHT: f32 = 0.15;
918    for (sample, energy) in out.iter_mut().zip(weight.iter()) {
919        if *energy > WOLA_MIN_WEIGHT {
920            *sample /= *energy;
921        }
922    }
923    // The head and tail lie outside any fully-stacked window and keep their original samples
924    // rather than a partially-normalized reconstruction.
925    let covered =
926        starts.first().copied().unwrap_or(0)..starts.last().map_or(0, |last| last + DENOISE_FRAME);
927    for (index, sample) in out.iter_mut().enumerate() {
928        if !covered.contains(&index) || weight[index] <= WOLA_MIN_WEIGHT {
929            *sample = pcm[index];
930        }
931    }
932    out
933}
934
935/// The exponential integral `E₁(x) = ∫ₓ^∞ e^{−t}/t dt`, for `x > 0`.
936///
937/// This is the term that makes MMSE-LSA gentle rather than gating: it grows without bound as the
938/// a priori SNR falls, so the log-amplitude estimate backs off smoothly instead of snapping shut.
939/// Abramowitz & Stegun 5.1.53 below 1 and 5.1.56 above it; both are accurate to ~2e-7, far inside
940/// what a spectral gain needs.
941fn exponential_integral_e1(x: f32) -> f32 {
942    if x <= 0.0 {
943        // ν ≤ 0 cannot arise from a non-negative SNR, but a denormal would otherwise return NaN
944        // and poison the frame.
945        return 0.0;
946    }
947    let x = f64::from(x);
948    let value = if x < 1.0 {
949        // A&S 5.1.53: E₁(x) + ln x = polynomial in x.
950        const A: [f64; 6] = [
951            -0.577_215_664_9,
952            0.999_991_93,
953            -0.249_910_55,
954            0.055_199_68,
955            -0.009_760_04,
956            0.001_078_57,
957        ];
958        let mut acc = 0.0;
959        for (power, coefficient) in A.iter().enumerate() {
960            acc += coefficient * x.powi(power as i32);
961        }
962        acc - x.ln()
963    } else {
964        // A&S 5.1.56: x·e^x·E₁(x) = rational in x.
965        const A: [f64; 4] = [8.573_328_74, 18.059_016_97, 8.634_760_89, 0.267_773_734];
966        const B: [f64; 4] = [9.573_322_34, 25.632_956_15, 21.099_653_08, 3.958_496_93];
967        let numerator = x.powi(4) + A[0] * x.powi(3) + A[1] * x * x + A[2] * x + A[3];
968        let denominator = x.powi(4) + B[0] * x.powi(3) + B[1] * x * x + B[2] * x + B[3];
969        (numerator / denominator) / (x * x.exp())
970    };
971    value as f32
972}
973
974/// Dereverberation runs its own STFT, deliberately coarser in time than the denoiser's.
975///
976/// The two want opposite things. Denoising wants short frames so a gain change lands inside a
977/// phoneme; prediction wants each frame to cover enough of the room's tail that a tractable
978/// number of taps can span it. At a 5.3 ms hop, a 24-tap filter reaches 128 ms — against an
979/// 810 ms reverb that removed 0.01 s of RT60, i.e. nothing (measured). A 10.7 ms hop with 40 taps
980/// reaches ~427 ms, which is the fraction of the tail single-channel prediction can model without
981/// the covariance becoming both enormous and ill-conditioned.
982const DEREVERB_FRAME: usize = 1024;
983const DEREVERB_HOP: usize = 256;
984
985/// Prediction taps: ~427 ms of tail at [`DEREVERB_HOP`].
986const DEREVERB_TAPS: usize = 40;
987
988/// Frames skipped before prediction starts, so the direct sound and its early reflections are
989/// never predictable from the regressor and therefore never subtracted. This delay is the whole
990/// reason WPE dereverberates instead of just whitening the voice.
991const DEREVERB_DELAY: usize = 2;
992
993/// Alternations between "estimate the speech variance" and "re-fit the filter". The variance
994/// estimate is what makes the fit ignore loud speech frames and key on the tail; two passes are
995/// enough to converge in practice, three leaves margin.
996const DEREVERB_ITERATIONS: usize = 3;
997
998/// Diagonal loading on the covariance, relative to its own trace. Silent bins are rank-deficient
999/// and would otherwise produce an arbitrary filter that injects noise instead of removing tail.
1000const DEREVERB_LOADING: f64 = 1e-4;
1001
1002/// What a `--dereverb` enrollment measured, so the CLI reports the effect rather than asserting it.
1003#[derive(Clone, Copy, Debug)]
1004pub struct DereverbReport {
1005    /// Reverberation time equivalent of the reference, before dereverberation.
1006    pub before_rt60_s: f32,
1007    /// The same measure afterwards.
1008    pub after_rt60_s: f32,
1009}
1010
1011/// Blind single-channel dereverberation by Weighted Prediction Error (Nakatani et al., 2010).
1012///
1013/// # Why this is a separate lever from `--denoise`
1014///
1015/// Reverb is *convolutive*: the microphone hears the voice convolved with the room's impulse
1016/// response. Denoising subtracts an *additive* stationary floor. The two do not overlap at all,
1017/// which is why running the denoiser on a reverberant reference moves the noise floor by 0.0 dB
1018/// and leaves the wetness untouched — measured, on exactly the recording that prompted this.
1019///
1020/// # Why it matters for enrollment specifically
1021///
1022/// The speaker encoder cannot separate voice from room, so a wet reference enrolls the room as
1023/// part of the speaker's identity and every utterance the clone speaks is rendered in that room
1024/// (measured: a 0.81 s reference produced a 0.79 s clone; a 0.66 s reference produced 0.68 s).
1025/// Drying the reference is therefore not cosmetic — it changes who the model thinks it is
1026/// imitating.
1027///
1028/// # The method
1029///
1030/// Late reverberation at frame `t` is, by construction, a linear function of the *past* of the
1031/// same signal: it is what earlier sound has decayed into. So per frequency bin, fit a linear
1032/// predictor from frames `t-D-L+1 ..= t-D` and subtract what it predicts. The delay `D` is what
1033/// protects the direct path: the speech itself is not predictable at that lag, the room's tail is.
1034///
1035/// The weighting is the "WPE" part and the reason it beats plain linear prediction. Each frame is
1036/// divided by the current estimate of the speech power there, so loud vowels — where the residual
1037/// is dominated by speech, not tail — stop dominating the fit. Estimating that power needs the
1038/// dereverberated signal, which needs the filter, so the two alternate for a few iterations.
1039///
1040/// Only late reverberation is removed. Early reflections arrive inside the protected delay by
1041/// design, so a very close, very live room is improved less than a distant one.
1042fn dereverb_reference(pcm: &[f32]) -> Vec<f32> {
1043    if pcm.len() < DEREVERB_FRAME * 4 {
1044        return pcm.to_vec();
1045    }
1046    let mut planner = rustfft::FftPlanner::<f32>::new();
1047    let forward = planner.plan_fft_forward(DEREVERB_FRAME);
1048    let inverse = planner.plan_fft_inverse(DEREVERB_FRAME);
1049
1050    let window: Vec<f32> = (0..DEREVERB_FRAME)
1051        .map(|n| {
1052            let phase = std::f32::consts::TAU * n as f32 / DEREVERB_FRAME as f32;
1053            0.5 - 0.5 * phase.cos()
1054        })
1055        .collect();
1056
1057    let bins = DEREVERB_FRAME / 2 + 1;
1058    let starts: Vec<usize> = (0..=pcm.len() - DEREVERB_FRAME)
1059        .step_by(DEREVERB_HOP)
1060        .collect();
1061    let frames = starts.len();
1062    if frames <= DEREVERB_DELAY + DEREVERB_TAPS + 2 {
1063        return pcm.to_vec();
1064    }
1065
1066    // Observed spectra, kept complex: prediction needs phase, unlike the magnitude-only denoiser.
1067    let mut observed: Vec<Vec<Complex64>> = Vec::with_capacity(frames);
1068    let mut scratch: Vec<rustfft::num_complex::Complex<f32>> =
1069        vec![rustfft::num_complex::Complex::new(0.0, 0.0); DEREVERB_FRAME];
1070    for &start in &starts {
1071        for (slot, n) in scratch.iter_mut().zip(0..DEREVERB_FRAME) {
1072            *slot = rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0);
1073        }
1074        forward.process(&mut scratch);
1075        observed.push(
1076            scratch[..bins]
1077                .iter()
1078                .map(|value| Complex64::new(f64::from(value.re), f64::from(value.im)))
1079                .collect(),
1080        );
1081    }
1082
1083    let mut desired = observed.clone();
1084    for _ in 0..DEREVERB_ITERATIONS {
1085        for bin in 0..bins {
1086            // Speech power per frame, from the current estimate. The floor keeps a silent frame
1087            // from receiving unbounded weight and hijacking the fit.
1088            let mut power: Vec<f64> = (0..frames).map(|t| desired[t][bin].norm_sqr()).collect();
1089            let mean = power.iter().sum::<f64>() / frames as f64;
1090            let floor = (mean * 1e-6).max(1e-12);
1091            for value in &mut power {
1092                *value = value.max(floor);
1093            }
1094
1095            let taps = DEREVERB_TAPS;
1096            let mut covariance = vec![Complex64::new(0.0, 0.0); taps * taps];
1097            let mut cross = vec![Complex64::new(0.0, 0.0); taps];
1098            for t in (DEREVERB_DELAY + taps)..frames {
1099                let weight = 1.0 / power[t];
1100                // Regressor: the observed signal at increasing lag past the protected delay.
1101                let regressor: Vec<Complex64> = (0..taps)
1102                    .map(|lag| observed[t - DEREVERB_DELAY - lag][bin])
1103                    .collect();
1104                for row in 0..taps {
1105                    let scaled = regressor[row] * weight;
1106                    for column in row..taps {
1107                        covariance[row * taps + column] += scaled * regressor[column].conj();
1108                    }
1109                    cross[row] += scaled * observed[t][bin].conj();
1110                }
1111            }
1112            // Hermitian: fill the lower triangle from the upper one that was accumulated.
1113            for row in 0..taps {
1114                for column in 0..row {
1115                    covariance[row * taps + column] = covariance[column * taps + row].conj();
1116                }
1117            }
1118            let trace: f64 = (0..taps).map(|i| covariance[i * taps + i].re).sum();
1119            if trace <= 0.0 {
1120                continue;
1121            }
1122            let loading = trace / taps as f64 * DEREVERB_LOADING;
1123            for i in 0..taps {
1124                covariance[i * taps + i] += Complex64::new(loading, 0.0);
1125            }
1126
1127            let Some(filter) = solve_complex_system(&mut covariance, &mut cross, taps) else {
1128                continue;
1129            };
1130            for t in 0..frames {
1131                if t < DEREVERB_DELAY + taps {
1132                    desired[t][bin] = observed[t][bin];
1133                    continue;
1134                }
1135                let mut tail = Complex64::new(0.0, 0.0);
1136                for (lag, coefficient) in filter.iter().enumerate() {
1137                    tail += coefficient.conj() * observed[t - DEREVERB_DELAY - lag][bin];
1138                }
1139                desired[t][bin] = observed[t][bin] - tail;
1140            }
1141        }
1142    }
1143
1144    // Overlap-add the dereverberated spectra back, mirroring the conjugate half so the inverse
1145    // transform yields a real signal.
1146    let mut out = vec![0.0_f32; pcm.len()];
1147    let mut weight = vec![0.0_f32; pcm.len()];
1148    for (index, &start) in starts.iter().enumerate() {
1149        let mut frame = vec![rustfft::num_complex::Complex::new(0.0_f32, 0.0); DEREVERB_FRAME];
1150        for bin in 0..bins {
1151            let value = desired[index][bin];
1152            #[allow(clippy::cast_possible_truncation)]
1153            let value = rustfft::num_complex::Complex::new(value.re as f32, value.im as f32);
1154            frame[bin] = value;
1155            let mirror = DEREVERB_FRAME - bin;
1156            if mirror != bin && mirror < DEREVERB_FRAME {
1157                frame[mirror] = value.conj();
1158            }
1159        }
1160        inverse.process(&mut frame);
1161        let scale = 1.0 / DEREVERB_FRAME as f32;
1162        for n in 0..DEREVERB_FRAME {
1163            out[start + n] += frame[n].re * scale * window[n];
1164            weight[start + n] += window[n] * window[n];
1165        }
1166    }
1167    for (sample, energy) in out.iter_mut().zip(weight.iter()) {
1168        if *energy > 1e-6 {
1169            *sample /= *energy;
1170        }
1171    }
1172    let covered =
1173        starts.first().copied().unwrap_or(0)..starts.last().map_or(0, |last| last + DEREVERB_FRAME);
1174    for (index, sample) in out.iter_mut().enumerate() {
1175        if !covered.contains(&index) || weight[index] <= 1e-6 {
1176            *sample = pcm[index];
1177        }
1178    }
1179    out
1180}
1181
1182/// Double-precision complex scalar for the normal equations.
1183///
1184/// The covariance is accumulated over thousands of frames and then inverted; doing that in f32
1185/// loses conditioning on quiet bins, which is where a bad filter does the most audible damage.
1186type Complex64 = rustfft::num_complex::Complex<f64>;
1187
1188/// Solves `a x = b` by Gaussian elimination with partial pivoting, consuming both.
1189///
1190/// Returns `None` when the system is singular to working precision, which the caller treats as
1191/// "leave this bin alone" rather than as a failure — a bin with no energy has no tail to remove.
1192fn solve_complex_system(
1193    a: &mut [Complex64],
1194    b: &mut [Complex64],
1195    n: usize,
1196) -> Option<Vec<Complex64>> {
1197    for column in 0..n {
1198        let (pivot, magnitude) = (column..n).fold((column, 0.0_f64), |best, row| {
1199            let candidate = a[row * n + column].norm_sqr();
1200            if candidate > best.1 {
1201                (row, candidate)
1202            } else {
1203                best
1204            }
1205        });
1206        if magnitude <= f64::MIN_POSITIVE {
1207            return None;
1208        }
1209        if pivot != column {
1210            for k in 0..n {
1211                a.swap(pivot * n + k, column * n + k);
1212            }
1213            b.swap(pivot, column);
1214        }
1215        let diagonal = a[column * n + column];
1216        for row in (column + 1)..n {
1217            let factor = a[row * n + column] / diagonal;
1218            if factor == Complex64::new(0.0, 0.0) {
1219                continue;
1220            }
1221            for k in column..n {
1222                let value = a[column * n + k] * factor;
1223                a[row * n + k] -= value;
1224            }
1225            let value = b[column] * factor;
1226            b[row] -= value;
1227        }
1228    }
1229    let mut solution = vec![Complex64::new(0.0, 0.0); n];
1230    for row in (0..n).rev() {
1231        let mut accumulator = b[row];
1232        for k in (row + 1)..n {
1233            accumulator -= a[row * n + k] * solution[k];
1234        }
1235        solution[row] = accumulator / a[row * n + row];
1236    }
1237    Some(solution)
1238}
1239
1240/// Reverberation time equivalent, in seconds, from the decay following speech offsets.
1241///
1242/// Reverb leaves no trace in a noise floor — what it does is stretch the energy envelope after
1243/// every stop. Measuring the median decay slope across offsets is what separates "the room rings"
1244/// from "the microphone hisses", two problems whose fixes have nothing in common. Returns `None`
1245/// when the audio has no clear offsets to measure.
1246fn reverb_time_s(pcm: &[f32]) -> Option<f32> {
1247    let hop = (SPEAKER_SAMPLE_RATE_HZ as usize) / 100; // 10 ms
1248    if pcm.len() < hop * 32 {
1249        return None;
1250    }
1251    let envelope: Vec<f32> = pcm
1252        .chunks_exact(hop)
1253        .map(|chunk| {
1254            let energy = chunk.iter().map(|s| s * s).sum::<f32>() / chunk.len() as f32;
1255            10.0 * (energy + 1e-9).log10()
1256        })
1257        .collect();
1258    let peak = envelope.iter().copied().fold(f32::NEG_INFINITY, f32::max);
1259    let span = 15_usize; // 150 ms of decay
1260    let mut slopes: Vec<f32> = Vec::new();
1261    for index in 1..envelope.len().saturating_sub(span) {
1262        if envelope[index] < peak - 25.0 || envelope[index] <= envelope[index - 1] {
1263            continue;
1264        }
1265        let drop = envelope[index] - envelope[index + span - 1];
1266        if drop < 6.0 {
1267            continue;
1268        }
1269        slopes.push(drop / (span as f32 * 0.01));
1270    }
1271    if slopes.is_empty() {
1272        return None;
1273    }
1274    slopes.sort_by(f32::total_cmp);
1275    let median = slopes[slopes.len() / 2];
1276    (median > 0.0).then(|| 60.0 / median)
1277}
1278
1279/// Root-mean-square of the quietest decile of 50 ms windows, in dBFS.
1280///
1281/// Whole-signal RMS hides pause noise behind the speech that dominates it, so enrollment
1282/// diagnostics report the floor between words — the part a denoise actually moves.
1283fn pause_floor_dbfs(pcm: &[f32]) -> f32 {
1284    let span = (SPEAKER_SAMPLE_RATE_HZ as usize) / 20;
1285    if pcm.len() < span {
1286        return f32::NEG_INFINITY;
1287    }
1288    let mut windows: Vec<f32> = pcm
1289        .chunks_exact(span)
1290        .map(|chunk| {
1291            (chunk
1292                .iter()
1293                .map(|s| f64::from(*s) * f64::from(*s))
1294                .sum::<f64>()
1295                / chunk.len() as f64)
1296                .sqrt() as f32
1297        })
1298        .filter(|rms| *rms > 0.0)
1299        .collect();
1300    if windows.is_empty() {
1301        return f32::NEG_INFINITY;
1302    }
1303    windows.sort_by(f32::total_cmp);
1304    let keep = (windows.len() / 10).max(1);
1305    let mean = windows[..keep].iter().sum::<f32>() / keep as f32;
1306    20.0 * mean.log10()
1307}
1308
1309/// One Lanczos tap: a sinc lowpass at `cutoff`, windowed by a wider sinc over `lobes`.
1310fn lanczos_tap(offset: f64, cutoff: f64, lobes: f64) -> f64 {
1311    let scaled = cutoff * offset;
1312    if scaled.abs() >= lobes {
1313        return 0.0;
1314    }
1315    sinc(scaled) * sinc(scaled / lobes)
1316}
1317
1318/// Normalized sinc, `sin(pi x) / (pi x)`, with the removable singularity at zero filled in.
1319fn sinc(x: f64) -> f64 {
1320    if x.abs() < 1e-12 {
1321        return 1.0;
1322    }
1323    let scaled = std::f64::consts::PI * x;
1324    scaled.sin() / scaled
1325}
1326
1327/// Hands the engine a `PreparedText` that was computed before the weights were borrowed.
1328struct PreparedPassThrough {
1329    prepared: PreparedText,
1330}
1331
1332impl TextPreparer for PreparedPassThrough {
1333    fn prepare(
1334        &self,
1335        _text: &str,
1336        _options: &NormalizationOptions,
1337    ) -> Result<PreparedText, TextPreparationError> {
1338        Ok(PreparedText::new(
1339            self.prepared.token_ids.clone(),
1340            NormalizationTrace {
1341                mode: self.prepared.normalization_trace.mode,
1342                unicode_version: self.prepared.normalization_trace.unicode_version.clone(),
1343                changes: self.prepared.normalization_trace.changes.clone(),
1344            },
1345        ))
1346    }
1347}
1348
1349/// A completed synthesis: the codes the talker produced and the audio they decode to.
1350pub struct SynthesizedAudio {
1351    /// Codec frames generated before the stop.
1352    pub frames: u64,
1353    /// Token ids that entered the model path, including the assistant wrapper.
1354    pub prepared_token_count: usize,
1355    /// Mono 24 kHz samples in `[-1, 1]`.
1356    pub pcm: Vec<f32>,
1357    /// Time from synthesis start (prompt work + prefill + first frames) to the first decoded
1358    /// packet of PCM existing. `None` when the run produced no audio. Time-to-first-audio and
1359    /// real-time factor are different products (doctrine: report them separately); this is the
1360    /// TTFA half, excluding model load, which the `load` stage event already bounds.
1361    pub ttfa: Option<std::time::Duration>,
1362}
1363
1364/// Run one utterance end to end: text, codes, PCM.
1365///
1366/// # Errors
1367///
1368/// Engine refusals (admission, budget, cancellation) and model refusals are mapped to their CLI
1369/// exit classes; a zero-frame generation is reported rather than written out as an empty file.
1370#[allow(clippy::too_many_arguments)]
1371pub fn synthesize(
1372    model: &LoadedModel,
1373    engine: &TtsEngine,
1374    request: &SynthesisRequest,
1375    speaker: &[f32],
1376    seed: u64,
1377    cancellation: &CancellationToken,
1378    observer: &dyn SynthesisObserver,
1379) -> Result<SynthesizedAudio, FttsError> {
1380    // 1. Text, once — see the module docs on ordering.
1381    let prepared_raw = model
1382        .tokenizer
1383        .prepare(&request.text, &request.normalization_options)
1384        .map_err(|error| FttsError::Input(format!("text preparation failed: {error}")))?;
1385    let wrapped = TalkerCheckpoint::wrap_target_ids(&prepared_raw.token_ids);
1386    let prepared = PreparedText::new(wrapped.clone(), prepared_raw.normalization_trace);
1387
1388    // 2. The cold-embedding rows this utterance can reach, and nothing else.
1389    let ids = TalkerCheckpoint::utterance_text_ids(&wrapped);
1390    let table = model
1391        .talker
1392        .gather_text_rows(&ids)
1393        .map_err(checkpoint_error)?;
1394
1395    // 3. The prompt header, derived from checkpoint tensors and the caller's speaker vector.
1396    let header = model
1397        .talker
1398        .xvector_header(&table, speaker, CODEC_LANGUAGE_ENGLISH_ID)
1399        .map_err(checkpoint_error)?;
1400    let tts_eos = model.talker.tts_eos(&table);
1401
1402    // 4. Borrowed weights for the generator.
1403    let talker_layers = model.talker.talker_layer_weights();
1404    let micro_layers = model.talker.microdecoder_layer_weights();
1405    let residual = model.talker.residual_embedding_slices();
1406    let heads = model.talker.microdecoder_head_slices();
1407    // The microdecoder's internal tables cover depths 2..=15: the first fourteen of the same
1408    // fifteen-table set the talker feedback path uses.
1409    let micro_residual = &residual[..residual.len() - 1];
1410
1411    let mut generator = QwenGenerator::new_with_artifact(
1412        QwenGeneratorConfig {
1413            talker_config: TalkerConfig::default(),
1414            talker_weights: model.talker.talker_weights(&talker_layers),
1415            text: model.talker.text_weights(&table),
1416            feedback: model.talker.feedback_tables(&residual),
1417            microdecoder_config: MicrodecoderConfig::default(),
1418            microdecoder_weights: model.talker.microdecoder_weights(
1419                &micro_layers,
1420                micro_residual,
1421                &heads,
1422            ),
1423            prompt_mode: PromptMode {
1424                clone_mode: CloneMode::XVector,
1425                non_streaming_mode: false,
1426            },
1427            header,
1428            tts_eos,
1429            reference: None,
1430            // The PRODUCT samples, exactly as the pinned upstream runtime does
1431            // (generation_config.json: do_sample=true, T=0.9, top_k=50, repetition_penalty=1.05,
1432            // subtalker likewise); canonical greedy remains the conformance decoder only. The p7r
1433            // forensics that certified this path: our talker draw stack matched torch's choices
1434            // code-for-code for seven straight frames from the same prefill, the silence defect was
1435            // the subtalker being forced greedy under a sampled talker (a measured silence
1436            // attractor the reference reproduces in that mismatched configuration), and with the
1437            // subtalker sampling per depth the engine's utterance envelope matches the reference's
1438            // sampled runs (peak frame RMS 0.086 with trailing silence). Determinism scope: build +
1439            // ISA + sampler version + seed, 16 draws per frame.
1440            sampling_mode: SamplingMode::Production,
1441            seed,
1442        },
1443        model.artifact.as_deref(),
1444    );
1445
1446    // 5. The engine owns admission, the budget, cancellation, and the frame loop — and the
1447    // codec decodes IN PARALLEL with it: a tee on the generator feeds every produced frame
1448    // through a bounded channel to a scoped codec worker driving the streaming decoder.
1449    // Streamed output is bit-identical to offline decode under every packet schedule (the
1450    // standing streaming==batch gate), so this overlap changes wall time and nothing else.
1451    // Deadlock shape: the worker only ever blocks on `recv` (it always drains), and the
1452    // generator only ever blocks on `send` when the worker is more than 256 frames behind —
1453    // bounded, and covered by the engine's rolling frame budget if the worker wedges.
1454    let preparer = PreparedPassThrough { prepared };
1455    let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<ftts_core::CodeFrame>(256);
1456    let codec = &model.codec;
1457    let synthesis_started = std::time::Instant::now();
1458    let (result, pcm, ttfa) = std::thread::scope(
1459        |scope| -> Result<
1460            (
1461                ftts_core::SynthesisResult,
1462                Vec<f32>,
1463                Option<std::time::Duration>,
1464            ),
1465            FttsError,
1466        > {
1467            let worker = scope.spawn(
1468                move || -> Result<(Vec<f32>, Option<std::time::Duration>), FttsError> {
1469                    // Overlap for real: this thread's int8 ops run serially on a spare core
1470                    // instead of contending for the generator's worker team.
1471                    ftts_kernels::team::bypass_team_on_this_thread();
1472                    const PACKET_FRAMES: usize = 4;
1473                    let mut state = codec.stream_state();
1474                    let mut pcm = Vec::new();
1475                    // `stream_push` REPLACES its output buffer with one packet's samples (see the
1476                    // streaming==offline test), so packets decode into a scratch and append here.
1477                    let mut packet_pcm = Vec::new();
1478                    let mut packet: Vec<i32> = Vec::with_capacity(16 * PACKET_FRAMES);
1479                    let mut packet_frames = 0_usize;
1480                    let mut first_audio_at: Option<std::time::Duration> = None;
1481                    while let Ok(frame) = frame_rx.recv() {
1482                        if frame.codes.len() != 16 {
1483                            return Err(FttsError::Generic(format!(
1484                                "generated frame carries {} codes, expected 16",
1485                                frame.codes.len()
1486                            )));
1487                        }
1488                        for code in &frame.codes {
1489                            packet.push(i32::try_from(*code).map_err(|_| {
1490                                FttsError::Generic(format!(
1491                                    "generated code {code} does not fit the codec's i32"
1492                                ))
1493                            })?);
1494                        }
1495                        packet_frames += 1;
1496                        if packet_frames == PACKET_FRAMES {
1497                            codec
1498                                .stream_push(&mut state, &packet, packet_frames, &mut packet_pcm)
1499                                .map_err(checkpoint_error)?;
1500                            pcm.extend_from_slice(&packet_pcm);
1501                            first_audio_at.get_or_insert_with(|| synthesis_started.elapsed());
1502                            packet.clear();
1503                            packet_frames = 0;
1504                        }
1505                    }
1506                    if packet_frames > 0 {
1507                        codec
1508                            .stream_push(&mut state, &packet, packet_frames, &mut packet_pcm)
1509                            .map_err(checkpoint_error)?;
1510                        pcm.extend_from_slice(&packet_pcm);
1511                        first_audio_at.get_or_insert_with(|| synthesis_started.elapsed());
1512                    }
1513                    Ok((pcm, first_audio_at))
1514                },
1515            );
1516
1517            let mut tee = TeeGenerator {
1518                inner: &mut generator,
1519                frames: frame_tx,
1520            };
1521            let result = engine
1522                .synthesize(
1523                    request.clone(),
1524                    &preparer,
1525                    &mut tee as &mut dyn FrameGenerator,
1526                    cancellation,
1527                    observer,
1528                )
1529                .map_err(engine_error);
1530            drop(tee); // closes the channel; the worker drains the tail packet and exits
1531            let pcm = worker.join().expect("codec worker must not panic");
1532            // The engine's error wins the report: when generation fails, the worker usually
1533            // fails too (starved or fed a partial stream), and its complaint would bury the
1534            // actual cause.
1535            let result = result?;
1536            let (pcm, ttfa) = pcm?;
1537            Ok((result, pcm, ttfa))
1538        },
1539    )?;
1540
1541    if result.code_frames.is_empty() {
1542        return Err(FttsError::Generic(
1543            "the talker stopped before emitting a frame; there is no audio to write. This is a \
1544             model or prompt problem, not an output problem — check the speaker vector and the \
1545             text"
1546                .to_owned(),
1547        ));
1548    }
1549
1550    Ok(SynthesizedAudio {
1551        frames: result.generated_frames,
1552        prepared_token_count: result.prepared_token_count,
1553        pcm,
1554        ttfa,
1555    })
1556}
1557
1558/// Forwards a generator's frames unchanged while teeing each one to the codec worker.
1559///
1560/// A send only fails when the worker has already died with its own error; surfacing a
1561/// generation error here aborts the engine loop early, and the worker's real failure is
1562/// reported at join.
1563struct TeeGenerator<'a> {
1564    inner: &'a mut dyn FrameGenerator,
1565    frames: std::sync::mpsc::SyncSender<ftts_core::CodeFrame>,
1566}
1567
1568impl FrameGenerator for TeeGenerator<'_> {
1569    fn begin_utterance(&mut self, prepared: &PreparedText) -> Result<(), GenerationError> {
1570        self.inner.begin_utterance(prepared)
1571    }
1572
1573    fn next_frame(&mut self) -> Result<Option<ftts_core::CodeFrame>, GenerationError> {
1574        let frame = self.inner.next_frame()?;
1575        if let Some(frame) = &frame
1576            && self.frames.send(frame.clone()).is_err()
1577        {
1578            return Err(GenerationError::new(
1579                "the codec worker stopped accepting frames; its error follows at join",
1580            ));
1581        }
1582        Ok(frame)
1583    }
1584}
1585
1586/// Map an engine refusal onto the CLI's exit-code contract.
1587fn engine_error(error: EngineError) -> FttsError {
1588    match error {
1589        EngineError::BudgetExceeded(_) => FttsError::BudgetTimeout(error.to_string()),
1590        EngineError::ResourceAdmission(_) => FttsError::BudgetTimeout(error.to_string()),
1591        EngineError::TextPreparation(_) => FttsError::Input(error.to_string()),
1592        other => FttsError::Generic(other.to_string()),
1593    }
1594}
1595
1596/// A model-side failure, for callers that need the engine's own error type.
1597#[must_use]
1598pub fn generation_error(message: &str) -> GenerationError {
1599    GenerationError::new(message)
1600}
1601
1602#[cfg(test)]
1603mod tests {
1604    use super::*;
1605
1606    /// Audio already at the pinned rate must come back untouched — the resample path is additive
1607    /// and may not perturb any enrollment that worked before it existed.
1608    #[test]
1609    fn audio_at_the_pinned_rate_is_returned_bit_for_bit() {
1610        let pcm: Vec<f32> = (0..4_096)
1611            .map(|n| (n as f32 * 0.017).sin() * 0.4 + (n as f32 * 0.31).sin() * 0.05)
1612            .collect();
1613        let out = resample_to_speaker_rate(pcm.clone(), SPEAKER_SAMPLE_RATE_HZ);
1614        assert_eq!(out.len(), pcm.len());
1615        for (index, (a, b)) in out.iter().zip(pcm.iter()).enumerate() {
1616            assert!(
1617                a.to_bits() == b.to_bits(),
1618                "sample {index} was altered at the pinned rate"
1619            );
1620        }
1621    }
1622
1623    /// `E₁` sets the MMSE-LSA gain at every bin of every frame, so a mistyped coefficient would
1624    /// quietly bias the whole denoiser instead of failing. References computed from the
1625    /// convergent series `−γ − ln x + Σ (−1)^{k+1} x^k /(k·k!)`, a different algorithm from the
1626    /// rational fits under test.
1627    ///
1628    /// The tolerance is tight on purpose: a single-digit slip in the fifth-order coefficient
1629    /// perturbs `E₁(0.9)` by ~7.6e-7, which a looser bound would wave through.
1630    #[test]
1631    fn the_exponential_integral_matches_its_series_expansion() {
1632        // (x, E₁(x)) — the series branch, x < 1.
1633        for (x, expected) in [
1634            (0.1_f32, 1.822_923_9_f32),
1635            (0.5, 0.559_773_6),
1636            (0.9, 0.260_183_94),
1637        ] {
1638            let actual = exponential_integral_e1(x);
1639            let relative = ((actual - expected) / expected).abs();
1640            assert!(
1641                relative < 1e-6,
1642                "E1({x}) = {actual} but the series gives {expected} (relative {relative:e})"
1643            );
1644        }
1645
1646        // E₁ is positive and strictly decreasing; the two branches must agree where they meet.
1647        let below = exponential_integral_e1(0.999_9);
1648        let above = exponential_integral_e1(1.000_1);
1649        assert!(
1650            below > above && (below - above).abs() < 1e-4,
1651            "the series and rational branches disagree across x = 1: {below} vs {above}"
1652        );
1653        assert_eq!(
1654            exponential_integral_e1(0.0),
1655            0.0,
1656            "a non-positive argument must not produce NaN"
1657        );
1658    }
1659
1660    /// The denoiser has to do both halves of its job: drop the noise floor between bursts, and
1661    /// leave the signal itself standing. A filter that achieves the first by attenuating
1662    /// everything would pass a floor-only check while destroying the voice it was meant to clean.
1663    #[test]
1664    fn denoise_lowers_the_floor_between_bursts_without_eating_the_signal() {
1665        const TONE_HZ: f64 = 700.0;
1666        let samples = SPEAKER_SAMPLE_RATE_HZ as usize * 2;
1667        // Deterministic hiss, so the assertion cannot flake on a lucky seed.
1668        let mut state = 0x2545_F491_4F6C_DD1D_u64;
1669        let mut noise = || {
1670            state ^= state << 13;
1671            state ^= state >> 7;
1672            state ^= state << 17;
1673            ((state >> 40) as f32 / 16_777_216.0) - 0.5
1674        };
1675
1676        // Half-second bursts of tone alternating with silence, all of it under hiss.
1677        let clean: Vec<f32> = (0..samples)
1678            .map(|n| {
1679                let t = n as f64 / f64::from(SPEAKER_SAMPLE_RATE_HZ);
1680                let speaking = (n / (SPEAKER_SAMPLE_RATE_HZ as usize / 2)).is_multiple_of(2);
1681                if speaking {
1682                    (std::f64::consts::TAU * TONE_HZ * t).sin() as f32 * 0.35
1683                } else {
1684                    0.0
1685                }
1686            })
1687            .collect();
1688        // Hiss at ~-26 dBFS against a 0.35 tone (~17 dB SNR): the audible-voice-memo regime
1689        // this lever exists for. (An earlier revision's generator bug made the "hiss" 4000x
1690        // louder than the signal, and the assertions below were calibrated against artifacts.)
1691        let noisy: Vec<f32> = clean.iter().map(|s| s + noise() * 0.1).collect();
1692
1693        let cleaned = denoise_reference(&noisy);
1694        assert_eq!(cleaned.len(), noisy.len(), "denoise must preserve length");
1695        assert!(
1696            cleaned.iter().all(|s| s.is_finite()),
1697            "denoise produced a non-finite sample"
1698        );
1699
1700        let before = pause_floor_dbfs(&noisy);
1701        let after = pause_floor_dbfs(&cleaned);
1702        assert!(
1703            after < before - 3.0,
1704            "expected the pause floor to drop by >3 dB, got {before:.1} -> {after:.1} dBFS"
1705        );
1706
1707        // Energy inside a burst must survive. Compare the loudest quarter-second of each.
1708        let span = SPEAKER_SAMPLE_RATE_HZ as usize / 4;
1709        let peak_rms = |pcm: &[f32]| {
1710            pcm.chunks_exact(span)
1711                .map(|c| (c.iter().map(|s| s * s).sum::<f32>() / c.len() as f32).sqrt())
1712                .fold(0.0_f32, f32::max)
1713        };
1714        let kept = peak_rms(&cleaned) / peak_rms(&noisy);
1715        assert!(
1716            kept > 0.7,
1717            "denoise removed too much of the signal: peak RMS kept {kept:.3} of the original"
1718        );
1719    }
1720
1721    /// A reference that opens on speech, with no leading room tone to learn from, must keep that
1722    /// opening — most voice memos start the moment recording does.
1723    ///
1724    /// The probe is deliberately voice-*like*: a harmonic stack with vibrato and an amplitude
1725    /// envelope. That matters, because a steady unvarying partial is spectrally what a hum is,
1726    /// and suppressing it is correct behaviour rather than a bug — an earlier version of this
1727    /// test used a bare sine and was measuring the denoiser doing its job.
1728    #[test]
1729    fn denoise_keeps_a_reference_that_opens_on_speech() {
1730        let rate = SPEAKER_SAMPLE_RATE_HZ as usize;
1731        let mut state = 0x9E37_79B9_7F4A_7C15_u64;
1732        let mut hiss = || {
1733            state ^= state << 13;
1734            state ^= state >> 7;
1735            state ^= state << 17;
1736            ((state >> 40) as f32 / 16_777_216.0) - 0.5
1737        };
1738        let burst = rate / 4;
1739        let noisy: Vec<f32> = (0..rate * 3)
1740            .map(|n| {
1741                let t = n as f64 / rate as f64;
1742                let index = n / burst;
1743                let voice = if index.is_multiple_of(2) {
1744                    // Vibrato and a syllable envelope keep the partials moving, which is what
1745                    // distinguishes a voice from a tone to any minimum/quantile noise estimator.
1746                    let vibrato = 1.0 + 0.03 * (std::f64::consts::TAU * 5.5 * t).sin();
1747                    let phase = (n % burst) as f32 / burst as f32;
1748                    let envelope = (std::f32::consts::PI * phase).sin();
1749                    let f0 = 140.0 * vibrato * (1.0 + 0.15 * (index / 2) as f64);
1750                    (1..=10)
1751                        .map(|h| {
1752                            let a = 0.3 / h as f32;
1753                            (std::f64::consts::TAU * f0 * h as f64 * t).sin() as f32 * a
1754                        })
1755                        .sum::<f32>()
1756                        * envelope
1757                } else {
1758                    0.0
1759                };
1760                voice + hiss() * 0.02
1761            })
1762            .collect();
1763
1764        let cleaned = denoise_reference(&noisy);
1765        let rms = |pcm: &[f32]| (pcm.iter().map(|s| s * s).sum::<f32>() / pcm.len() as f32).sqrt();
1766        let kept = rms(&cleaned[..burst]) / rms(&noisy[..burst]);
1767        assert!(
1768            kept > 0.7,
1769            "the opening burst kept only {kept:.3} of its energy; the noise floor is being \
1770             seeded from speech the estimator has not yet learned to exclude"
1771        );
1772    }
1773
1774    /// Denoising must not buy a quiet floor by dulling the voice.
1775    ///
1776    /// High frequencies are where this fails first and where it matters most: broadband hiss
1777    /// overlaps sibilance almost exactly, so a suppressor tuned by overall SNR happily trades
1778    /// away 4–10 kHz — and the speaker encoder reads sibilance as identity, so that trade shows
1779    /// up as a duller *and less recognizable* clone rather than merely a duller one.
1780    ///
1781    /// The probe is broadband speech-like content: every band must survive comparably, so the
1782    /// assertion is on the spread across bands, not on any single band's absolute retention.
1783    #[test]
1784    fn denoise_does_not_preferentially_zap_high_frequencies() {
1785        let rate = SPEAKER_SAMPLE_RATE_HZ as usize;
1786        let mut state = 0xDEAD_BEEF_1234_5678_u64;
1787        let mut hiss = || {
1788            state ^= state << 13;
1789            state ^= state >> 7;
1790            state ^= state << 17;
1791            ((state >> 40) as f32 / 16_777_216.0) - 0.5
1792        };
1793        // Equal-amplitude tones spanning the band, gated into syllable-like bursts so the
1794        // estimator treats them as speech rather than as hum.
1795        let probes: [f64; 5] = [300.0, 1_200.0, 3_000.0, 6_000.0, 9_000.0];
1796        let burst = rate / 4;
1797        let noisy: Vec<f32> = (0..rate * 3)
1798            .map(|n| {
1799                let t = n as f64 / rate as f64;
1800                let voice = if (n / burst).is_multiple_of(2) {
1801                    let phase = (n % burst) as f32 / burst as f32;
1802                    let envelope = (std::f32::consts::PI * phase).sin();
1803                    probes
1804                        .iter()
1805                        .map(|hz| (std::f64::consts::TAU * hz * t).sin() as f32 * 0.12)
1806                        .sum::<f32>()
1807                        * envelope
1808                } else {
1809                    0.0
1810                };
1811                voice + hiss() * 0.02
1812            })
1813            .collect();
1814
1815        let cleaned = denoise_reference(&noisy);
1816
1817        // Per-probe retention, measured by projecting each band onto its own tone (a one-bin
1818        // Goertzel-style correlation) inside a burst.
1819        let span = burst / 2..burst;
1820        let energy_at = |pcm: &[f32], hz: f64| -> f32 {
1821            let (mut re, mut im) = (0.0_f64, 0.0_f64);
1822            for (offset, sample) in pcm[span.clone()].iter().enumerate() {
1823                let t = (span.start + offset) as f64 / rate as f64;
1824                let angle = std::f64::consts::TAU * hz * t;
1825                re += f64::from(*sample) * angle.cos();
1826                im += f64::from(*sample) * angle.sin();
1827            }
1828            (re.hypot(im) / span.len() as f64) as f32
1829        };
1830
1831        let retention: Vec<f32> = probes
1832            .iter()
1833            .map(|hz| energy_at(&cleaned, *hz) / energy_at(&noisy, *hz).max(1e-9))
1834            .collect();
1835        let low = retention[0];
1836        for (hz, kept) in probes.iter().zip(retention.iter()) {
1837            assert!(
1838                *kept > 0.5,
1839                "{hz} Hz retained only {kept:.3}; the denoiser is eating the band, not the noise \
1840                 (all bands: {retention:?})"
1841            );
1842            assert!(
1843                *kept > low * 0.6,
1844                "{hz} Hz retained {kept:.3} against {low:.3} at 300 Hz — high frequencies are \
1845                 being attenuated preferentially, which is how sibilance and speaker identity go \
1846                 (all bands: {retention:?})"
1847            );
1848        }
1849    }
1850
1851    /// A clip too short to survive its own downsample must round to nothing here rather than
1852    /// reaching the mel front end as an empty slice — which is why `decode_reference_audio`
1853    /// re-checks emptiness against the resampled PCM instead of only the decoded PCM.
1854    #[test]
1855    fn a_clip_shorter_than_its_downsample_ratio_resamples_to_nothing() {
1856        let out = resample_to_speaker_rate(vec![0.25], 192_000);
1857        assert!(
1858            out.is_empty(),
1859            "one sample at 192 kHz is less than half an output sample at \
1860             {SPEAKER_SAMPLE_RATE_HZ} Hz, so it cannot produce one"
1861        );
1862    }
1863
1864    /// A tone that survives the resample proves the kernel is a real lowpass and not a decimator:
1865    /// 48 kHz is the rate every phone and Mac voice memo records at, and a 1 kHz tone sits well
1866    /// inside the 12 kHz band that survives the trip to 24 kHz.
1867    #[test]
1868    fn a_48k_tone_resamples_to_24k_with_its_shape_intact() {
1869        const SOURCE_HZ: u32 = 48_000;
1870        const TONE_HZ: f64 = 1_000.0;
1871        let samples = SOURCE_HZ as usize; // one second
1872        let pcm: Vec<f32> = (0..samples)
1873            .map(|n| {
1874                let t = n as f64 / f64::from(SOURCE_HZ);
1875                (std::f64::consts::TAU * TONE_HZ * t).sin() as f32
1876            })
1877            .collect();
1878
1879        let out = resample_to_speaker_rate(pcm, SOURCE_HZ);
1880
1881        let expected_len = SPEAKER_SAMPLE_RATE_HZ as usize;
1882        assert!(
1883            out.len().abs_diff(expected_len) <= 1,
1884            "expected ~{expected_len} samples at {SPEAKER_SAMPLE_RATE_HZ} Hz, got {}",
1885            out.len()
1886        );
1887
1888        // Compare against the tone sampled directly at the target rate, ignoring the window's
1889        // run-up at each end where the kernel is truncated by the signal boundary.
1890        let skip = 64;
1891        let interior = out.len() - skip;
1892        let mut worst = 0.0_f32;
1893        for (index, sample) in out.iter().enumerate().take(interior).skip(skip) {
1894            let t = index as f64 / f64::from(SPEAKER_SAMPLE_RATE_HZ);
1895            let ideal = (std::f64::consts::TAU * TONE_HZ * t).sin() as f32;
1896            worst = worst.max((sample - ideal).abs());
1897        }
1898        assert!(
1899            worst < 0.02,
1900            "resampled tone drifted from the analytic reference by {worst}"
1901        );
1902    }
1903
1904    #[test]
1905    fn a_short_speaker_vector_is_refused_rather_than_padded() {
1906        let dir = std::env::temp_dir().join("ftts-synth-tests");
1907        fs::create_dir_all(&dir).expect("temp dir");
1908        let path = dir.join("short.spk");
1909        fs::write(&path, vec![0u8; 64]).expect("write");
1910        let error = read_speaker_vector(&path).expect_err("a short vector must be refused");
1911        let message = error.to_string();
1912        assert!(message.contains("64 bytes"), "{message}");
1913        assert!(message.contains("4096"), "{message}");
1914    }
1915
1916    #[test]
1917    fn a_non_finite_speaker_vector_is_refused() {
1918        let dir = std::env::temp_dir().join("ftts-synth-tests");
1919        fs::create_dir_all(&dir).expect("temp dir");
1920        let path = dir.join("nan.spk");
1921        let mut bytes = vec![0u8; SPEAKER_VECTOR_BYTES];
1922        bytes[0..4].copy_from_slice(&f32::NAN.to_le_bytes());
1923        fs::write(&path, &bytes).expect("write");
1924        let error = read_speaker_vector(&path).expect_err("NaN must be refused");
1925        assert!(error.to_string().contains("index 0"), "{error}");
1926    }
1927
1928    #[test]
1929    fn a_well_formed_speaker_vector_reads_back_exactly() {
1930        let dir = std::env::temp_dir().join("ftts-synth-tests");
1931        fs::create_dir_all(&dir).expect("temp dir");
1932        let path = dir.join("good.spk");
1933        let expected: Vec<f32> = (0..TALKER_HIDDEN).map(|i| i as f32 * 0.001).collect();
1934        let mut bytes = Vec::with_capacity(SPEAKER_VECTOR_BYTES);
1935        for value in &expected {
1936            bytes.extend_from_slice(&value.to_le_bytes());
1937        }
1938        fs::write(&path, &bytes).expect("write");
1939        assert_eq!(read_speaker_vector(&path).expect("read"), expected);
1940    }
1941
1942    #[test]
1943    fn enrollment_writer_refuses_overwrite_and_preserves_the_vector() {
1944        let path = std::env::temp_dir().join(format!(
1945            "ftts-enroll-{}-{}.spk",
1946            std::process::id(),
1947            std::time::SystemTime::now()
1948                .duration_since(std::time::UNIX_EPOCH)
1949                .expect("clock")
1950                .as_nanos()
1951        ));
1952        let expected: Vec<f32> = (0..TALKER_HIDDEN)
1953            .map(|index| index as f32 * 0.125)
1954            .collect();
1955        write_speaker_vector_new(&path, &expected).expect("initial enrollment write");
1956        assert_eq!(
1957            read_speaker_vector(&path).expect("read enrolled vector"),
1958            expected
1959        );
1960        let error = write_speaker_vector_new(&path, &[0.0; TALKER_HIDDEN])
1961            .expect_err("an enrollment must never replace an existing voice");
1962        assert!(error.to_string().contains("without overwriting"), "{error}");
1963    }
1964
1965    #[test]
1966    fn wav_reference_decodes_to_mono_24khz_pcm() {
1967        let path = std::env::temp_dir().join(format!(
1968            "ftts-reference-{}-{}.wav",
1969            std::process::id(),
1970            std::time::SystemTime::now()
1971                .duration_since(std::time::UNIX_EPOCH)
1972                .expect("clock")
1973                .as_nanos()
1974        ));
1975        let pcm: Vec<f32> = (0..1_920)
1976            .map(|index| (index as f32 / 1_920.0 * std::f32::consts::TAU).sin() * 0.25)
1977            .collect();
1978        fs::write(
1979            &path,
1980            ftts_core::audio::encode_wav(&pcm, SPEAKER_SAMPLE_RATE_HZ),
1981        )
1982        .expect("write reference WAV");
1983        let decoded = decode_reference_audio(&path).expect("decode reference WAV");
1984        assert_eq!(decoded.len(), pcm.len());
1985        assert!(decoded.iter().all(|sample| sample.is_finite()));
1986    }
1987
1988    #[test]
1989    fn a_bundle_names_the_file_that_is_actually_missing() {
1990        // An agent that gets "model not found" for a directory holding three of four files cannot
1991        // act on it; the message must name the one that is absent.
1992        let dir = std::env::temp_dir().join("ftts-bundle-tests-empty");
1993        fs::create_dir_all(&dir).expect("temp dir");
1994        let error = ModelBundle::resolve(&dir).expect_err("an empty directory is not a bundle");
1995        assert!(error.to_string().contains("model.safetensors"), "{error}");
1996    }
1997
1998    #[test]
1999    fn a_complete_bundle_prefers_its_canonical_artifact_for_synthesis() {
2000        let nonce = std::time::SystemTime::now()
2001            .duration_since(std::time::UNIX_EPOCH)
2002            .expect("clock after epoch")
2003            .as_nanos();
2004        let dir = std::env::temp_dir().join(format!(
2005            "ftts-bundle-canonical-{}-{nonce}",
2006            std::process::id()
2007        ));
2008        fs::create_dir_all(dir.join("speech_tokenizer")).expect("create bundle sidecar directory");
2009        for name in [
2010            CANONICAL_MODEL_BASENAME,
2011            "speech_tokenizer/model.safetensors",
2012            "vocab.json",
2013            "merges.txt",
2014            "tokenizer_config.json",
2015        ] {
2016            fs::write(dir.join(name), []).expect("write bundle fixture sidecar");
2017        }
2018
2019        let expected_artifact = dir.join(CANONICAL_MODEL_BASENAME);
2020        let bundle = ModelBundle::resolve(&dir).expect("complete canonical bundle resolves");
2021        assert_eq!(
2022            bundle.canonical_main.as_deref(),
2023            Some(expected_artifact.as_path())
2024        );
2025        assert!(
2026            !bundle.main.exists(),
2027            "canonical synthesis must not require the raw main checkpoint"
2028        );
2029
2030        let explicit = ModelBundle::resolve(&expected_artifact)
2031            .expect("an explicit canonical artifact resolves against its sidecars");
2032        assert_eq!(
2033            explicit.canonical_main.as_deref(),
2034            Some(expected_artifact.as_path())
2035        );
2036    }
2037}