Skip to main content

ftts_cli/
synth.rs

1//! The `ftts say` synthesis path: text in, 24 kHz PCM out.
2//!
3//! This module is where the CLI stops describing the pipeline and runs it. It resolves a
4//! checkpoint bundle, hydrates the talker and the codec, tokenizes and wraps the text, derives the
5//! prompt header, drives [`TtsEngine::synthesize`] over the real [`QwenGenerator`], and hands the
6//! generated codes to the codec decoder. What comes back is `f32` samples; the WAV writing lives
7//! in `ftts-core::audio` and the sink policy in [`crate::AudioOutput`].
8//!
9//! # Why the text is prepared before the engine runs
10//!
11//! [`TtsEngine::synthesize`] owns text preparation, and normally that is where tokenization
12//! happens. Here it happens once, up front, and the engine is handed a preparer that returns that
13//! exact result. The reason is the cold text embedding: it is `[151936, 2048]`, and materializing
14//! it whole to serve a fifteen-token utterance would cost 1.24 GB. The gather needs the token ids,
15//! the generator needs the gathered table, and the generator must exist before `synthesize` is
16//! called — so the ids have to be known first. The engine still receives, verbatim, the
17//! `PreparedText` a fresh call would have produced; nothing is skipped, only ordered.
18//!
19//! # Speaker conditioning is derived, never invented
20//!
21//! An x-vector prompt conditions on a 1,024-wide speaker embedding. A voice source may be either
22//! a precomputed raw vector (1,024 little-endian `f32`, 4,096 bytes) or reference audio decoded
23//! through the pinned 24 kHz log-mel front end and ECAPA encoder. Neither path accepts a
24//! fabricated vector.
25
26use crate::error::FttsError;
27use ftts_core::{
28    CancellationToken, EngineError, FrameGenerator, GenerationError, NormalizationOptions,
29    NormalizationTrace, PreparedText, SynthesisObserver, SynthesisRequest, TextPreparationError,
30    TextPreparer, TtsEngine,
31};
32use ftts_model_qwen::checkpoint::{
33    CODEC_LANGUAGE_ENGLISH_ID, CheckpointError, CodecCheckpoint, TALKER_HIDDEN, TalkerCheckpoint,
34};
35use ftts_model_qwen::generate::{QwenGenerator, QwenGeneratorConfig};
36use ftts_model_qwen::microdecoder::MicrodecoderConfig;
37use ftts_model_qwen::prompt::{CloneMode, PromptMode};
38use ftts_model_qwen::sampler::SamplingMode;
39use ftts_model_qwen::speaker::{
40    Encoder as SpeakerEncoder, SPEAKER_SAMPLE_RATE_HZ, log_mel_from_24khz_pcm,
41};
42use ftts_model_qwen::talker::TalkerConfig;
43use ftts_model_qwen::tokenizer::{QwenTokenizer, TokenizerFiles};
44use std::fs;
45use std::fs::OpenOptions;
46use std::path::{Path, PathBuf};
47use symphonia::core::audio::SampleBuffer;
48use symphonia::core::codecs::DecoderOptions;
49use symphonia::core::errors::Error as SymphoniaError;
50use symphonia::core::formats::FormatOptions;
51use symphonia::core::io::MediaSourceStream;
52use symphonia::core::meta::MetadataOptions;
53use symphonia::core::probe::Hint;
54use symphonia::default::{get_codecs, get_probe};
55
56/// Bytes in a speaker-vector file: 1,024 little-endian `f32`.
57pub const SPEAKER_VECTOR_BYTES: usize = TALKER_HIDDEN * 4;
58
59const CANONICAL_MODEL_BASENAME: &str = "qwen3-tts-12hz-0.6b-base.fttsq";
60
61fn checkpoint_error(error: CheckpointError) -> FttsError {
62    FttsError::ArtifactFormat(error.to_string())
63}
64
65/// The model resources `ftts say` needs, located relative to one model path.
66#[derive(Clone, Debug)]
67pub struct ModelBundle {
68    /// Directory holding the artifact sidecars and tokenizer files.
69    pub root: PathBuf,
70    /// The raw main checkpoint, retained for enrollment-only components that have not yet gained
71    /// a canonical-artifact accessor.
72    pub main: PathBuf,
73    /// The portable main-weight artifact selected for synthesis, when present.
74    pub canonical_main: Option<PathBuf>,
75    /// The codec decoder checkpoint.
76    pub codec: PathBuf,
77}
78
79impl ModelBundle {
80    /// Resolve a bundle from `--model`, which may name the directory, a `.fttsq`, or
81    /// `model.safetensors`.
82    ///
83    /// # Errors
84    ///
85    /// [`FttsError::ModelNotFound`] naming the exact missing file, so a partial download is
86    /// diagnosable without guessing which of the four is absent.
87    pub fn resolve(model: &Path) -> Result<Self, FttsError> {
88        let root = if model.is_dir() {
89            model.to_path_buf()
90        } else {
91            model
92                .parent()
93                .ok_or_else(|| {
94                    FttsError::ModelNotFound(format!(
95                        "model path {} has no parent directory",
96                        model.display()
97                    ))
98                })?
99                .to_path_buf()
100        };
101        let canonical_main = if model.is_dir() {
102            let canonical = root.join(CANONICAL_MODEL_BASENAME);
103            if canonical.is_file() {
104                Some(canonical)
105            } else {
106                None
107            }
108        } else if model.extension().and_then(|extension| extension.to_str()) == Some("fttsq") {
109            Some(model.to_path_buf())
110        } else {
111            None
112        };
113        let main = root.join("model.safetensors");
114        let codec = root.join("speech_tokenizer/model.safetensors");
115        let (main_label, main_path) = match canonical_main.as_ref() {
116            Some(path) => ("canonical talker artifact", path),
117            None => ("talker checkpoint", &main),
118        };
119        for (label, path) in [
120            (main_label, main_path),
121            ("codec checkpoint", &codec),
122            ("tokenizer vocabulary", &root.join("vocab.json")),
123            ("tokenizer merges", &root.join("merges.txt")),
124            ("tokenizer config", &root.join("tokenizer_config.json")),
125        ] {
126            if !path.is_file() {
127                return Err(FttsError::ModelNotFound(format!(
128                    "{label} is missing at {}; `ftts say` needs a complete model directory \
129                     ({CANONICAL_MODEL_BASENAME} or model.safetensors, \
130                     speech_tokenizer/model.safetensors, \
131                     vocab.json, merges.txt, tokenizer_config.json)",
132                    path.display()
133                )));
134            }
135        }
136        Ok(Self {
137            root,
138            main,
139            canonical_main,
140            codec,
141        })
142    }
143}
144
145/// Every weight and table one `say` needs, hydrated once.
146pub struct LoadedModel {
147    talker: TalkerCheckpoint,
148    codec: CodecCheckpoint,
149    tokenizer: QwenTokenizer,
150    /// The checkpoint's own digest-verified mapping of the canonical artifact, shared so the
151    /// int8 route hydrates its Q8 tables from it (proven byte-identical to requantizing the
152    /// widened f32 copies, scales included). Never re-opened: every `MappedFttsq::open`
153    /// re-verifies the whole artifact's digests, ~1.3 GB of hashing.
154    artifact: Option<std::sync::Arc<ftts_artifacts::fttsq::MappedFttsq>>,
155}
156
157impl LoadedModel {
158    /// Hydrate the bundle. This reads gigabytes and is the slow step of a cold run.
159    ///
160    /// # Errors
161    ///
162    /// If any checkpoint or tokenizer file is unreadable or not the pinned model.
163    pub fn load(bundle: &ModelBundle) -> Result<Self, FttsError> {
164        let read = |name: &str| -> Result<String, FttsError> {
165            let path = bundle.root.join(name);
166            fs::read_to_string(&path).map_err(|error| {
167                FttsError::ArtifactFormat(format!("cannot read {}: {error}", path.display()))
168            })
169        };
170        let vocab = read("vocab.json")?;
171        let merges = read("merges.txt")?;
172        let config = read("tokenizer_config.json")?;
173
174        // The three heavyweight hydrations are independent, so the codec checkpoint and the
175        // tokenizer build overlap the talker load instead of queueing behind it. Each result is
176        // computed exactly as it was serially; only wall time changes.
177        let (talker, codec, tokenizer) = std::thread::scope(|scope| {
178            let codec = scope.spawn(|| CodecCheckpoint::load(&bundle.codec));
179            let tokenizer = scope.spawn(|| {
180                QwenTokenizer::from_files_using_environment(TokenizerFiles {
181                    vocab_json: &vocab,
182                    merges_txt: &merges,
183                    tokenizer_config_json: &config,
184                })
185            });
186            let talker = match bundle.canonical_main.as_deref() {
187                // The elision mirrors the generator's own hydration decision for this process:
188                // stacks that will run artifact-native int8 skip their dead f32 projections
189                // (~2.5 GB of widening nobody reads).
190                Some(path) => TalkerCheckpoint::load_fttsq_elided(
191                    path,
192                    ftts_model_qwen::generate::hot_elision_from_environment(),
193                ),
194                None => TalkerCheckpoint::load(&bundle.main),
195            };
196            (
197                talker,
198                codec.join().expect("codec loader panicked"),
199                tokenizer.join().expect("tokenizer builder panicked"),
200            )
201        });
202        let tokenizer = tokenizer
203            .map_err(|error| FttsError::ArtifactFormat(format!("tokenizer unusable: {error}")))?;
204
205        let talker = talker.map_err(checkpoint_error)?;
206        // Shared, not re-opened: a second MappedFttsq::open would re-verify the whole artifact's
207        // digests (~1.3 GB of hashing) for a mapping the checkpoint already carries.
208        let artifact = talker.artifact().cloned();
209        Ok(Self {
210            talker,
211            codec: codec.map_err(checkpoint_error)?,
212            tokenizer,
213            artifact,
214        })
215    }
216}
217
218/// Read a precomputed 1,024-wide speaker vector.
219///
220/// See the module docs on why this is a raw vector rather than a `.ftvoice` pack.
221///
222/// # Errors
223///
224/// [`FttsError::Input`] when the file is unreadable or is not exactly
225/// [`SPEAKER_VECTOR_BYTES`] bytes — a truncated vector would otherwise be padded with silence and
226/// change the voice in a way only listening could detect.
227pub fn read_speaker_vector(path: &Path) -> Result<Vec<f32>, FttsError> {
228    let bytes = fs::read(path).map_err(|error| {
229        FttsError::Input(format!(
230            "cannot read speaker vector {}: {error}",
231            path.display()
232        ))
233    })?;
234    if bytes.len() != SPEAKER_VECTOR_BYTES {
235        return Err(FttsError::Input(format!(
236            "speaker vector {} is {} bytes; `ftts say --voice` expects exactly {} \
237             ({TALKER_HIDDEN} little-endian f32)",
238            path.display(),
239            bytes.len(),
240            SPEAKER_VECTOR_BYTES
241        )));
242    }
243    let vector: Vec<f32> = bytes
244        .as_chunks::<4>()
245        .0
246        .iter()
247        .map(|quad| f32::from_le_bytes(*quad))
248        .collect();
249    if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
250        return Err(FttsError::Input(format!(
251            "speaker vector {} holds a non-finite value at index {index}; it would poison every \
252             prefill position it is summed into",
253            path.display()
254        )));
255    }
256    Ok(vector)
257}
258
259/// Derive an x-vector from an enrolled raw vector or a real reference recording.
260/// What a `--denoise` enrollment measured, so the CLI can report the effect rather than assert it.
261#[derive(Clone, Copy, Debug)]
262pub struct DenoiseReport {
263    /// Pause floor of the decoded reference, before denoising.
264    pub before_dbfs: f32,
265    /// Pause floor after denoising.
266    pub after_dbfs: f32,
267}
268
269/// Which reference-cleanup stages to run, and where each reports what it measured.
270///
271/// Both default to off. Every stage changes the enrolled identity, so none of them is applied on
272/// a user's behalf — a lever that can alter who the clone sounds like is opted into by name.
273#[derive(Default)]
274pub struct ReferenceCleanup<'a> {
275    /// Spectral-subtract the stationary noise floor.
276    pub denoise: Option<&'a mut Option<DenoiseReport>>,
277    /// Remove late reverberation.
278    pub dereverb: Option<&'a mut Option<DereverbReport>>,
279}
280
281/// Derive a speaker vector from a voice source: a raw x-vector file, or reference audio.
282///
283/// Passing `Some` for a cleanup slot opts the reference into that stage and fills the slot with
284/// what it measured. Dereverberation runs first: it is a linear operation on the observed signal,
285/// so applying it before the noise floor is estimated keeps that estimate from being fitted to a
286/// signal the next stage is about to change.
287///
288/// # Errors
289///
290/// When the source cannot be read, decoded, resampled, or encoded into a finite x-vector.
291pub fn speaker_from_voice(
292    bundle: &ModelBundle,
293    path: &Path,
294    cleanup: ReferenceCleanup<'_>,
295) -> Result<Vec<f32>, FttsError> {
296    let bytes = fs::read(path).map_err(|error| {
297        FttsError::Input(format!(
298            "cannot read voice source {}: {error}",
299            path.display()
300        ))
301    })?;
302    // Size alone must not decide: a perfectly plausible 4,096-byte audio clip would
303    // otherwise reinterpret as garbage floats and enroll a nonsense voice without any
304    // error. A recognizable audio container magic wins over the size sniff.
305    let looks_like_audio = bytes.len() >= 12
306        && (bytes.starts_with(b"RIFF")
307            || bytes.starts_with(b"fLaC")
308            || bytes.starts_with(b"ID3")
309            || bytes.starts_with(b"OggS")
310            || &bytes[4..8] == b"ftyp");
311    if bytes.len() == SPEAKER_VECTOR_BYTES && !looks_like_audio {
312        return decode_speaker_vector(path, &bytes);
313    }
314    let pcm = decode_reference_audio_any(path)?;
315    let ReferenceCleanup { denoise, dereverb } = cleanup;
316    let pcm = match dereverb {
317        Some(report) => {
318            let before = reverb_time_s(&pcm);
319            let dried = dereverb_reference(&pcm);
320            let after = reverb_time_s(&dried);
321            if let (Some(before), Some(after)) = (before, after) {
322                *report = Some(DereverbReport {
323                    before_rt60_s: before,
324                    after_rt60_s: after,
325                });
326            }
327            dried
328        }
329        None => pcm,
330    };
331    let pcm = match denoise {
332        Some(report) => {
333            let before = pause_floor_dbfs(&pcm);
334            let cleaned = match neural_denoise_reference(bundle, &pcm)? {
335                Some(cleaned) => cleaned,
336                None => denoise_reference(&pcm),
337            };
338            *report = Some(DenoiseReport {
339                before_dbfs: before,
340                after_dbfs: pause_floor_dbfs(&cleaned),
341            });
342            cleaned
343        }
344        None => pcm,
345    };
346    let mel = log_mel_from_24khz_pcm(&pcm)
347        .map_err(|error| FttsError::Input(format!("cannot extract speaker features: {error}")))?;
348    let encoder = match bundle.canonical_main.as_deref() {
349        Some(artifact) => SpeakerEncoder::load_fttsq(artifact),
350        None => SpeakerEncoder::load(&bundle.main),
351    }
352    .map_err(checkpoint_error)?;
353    let vector = encoder.encode(&mel.values, mel.frames);
354    if vector.iter().all(|value| value.is_finite()) {
355        Ok(vector)
356    } else {
357        Err(FttsError::Input(
358            "speaker encoder produced a non-finite x-vector; refusing to condition synthesis"
359                .to_owned(),
360        ))
361    }
362}
363
364/// Write a raw x-vector without replacing an existing enrollment result.
365pub fn write_speaker_vector_new(path: &Path, vector: &[f32]) -> Result<(), FttsError> {
366    if vector.len() != TALKER_HIDDEN {
367        return Err(FttsError::Input(format!(
368            "cannot write {}-wide speaker vector; expected {TALKER_HIDDEN}",
369            vector.len()
370        )));
371    }
372    if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
373        return Err(FttsError::Input(format!(
374            "cannot write speaker vector with a non-finite value at index {index}"
375        )));
376    }
377    let mut bytes = Vec::with_capacity(SPEAKER_VECTOR_BYTES);
378    for value in vector {
379        bytes.extend_from_slice(&value.to_le_bytes());
380    }
381    use std::io::Write;
382    let mut file = OpenOptions::new()
383        .write(true)
384        .create_new(true)
385        .open(path)
386        .map_err(|error| {
387            FttsError::Input(format!(
388                "cannot create enrolled voice {} without overwriting an existing file: {error}",
389                path.display()
390            ))
391        })?;
392    file.write_all(&bytes).map_err(|error| {
393        FttsError::Input(format!(
394            "cannot write enrolled voice {}: {error}",
395            path.display()
396        ))
397    })
398}
399
400/// Replaces an existing enrolled voice, keeping the displaced one alongside it.
401///
402/// Enrollment is cheap to redo but a voice is not always cheap to re-record, and the reference a
403/// `.spk` came from may be long gone. The previous vector is copied to `<path>.bak` before the new
404/// one lands, so a mistaken overwrite is one `mv` away from undone rather than unrecoverable.
405///
406/// # Errors
407///
408/// When the vector is malformed, the backup cannot be written, or the file cannot be replaced.
409pub fn replace_speaker_vector(path: &Path, vector: &[f32]) -> Result<PathBuf, FttsError> {
410    let backup = path.with_extension("spk.bak");
411    fs::copy(path, &backup).map_err(|error| {
412        FttsError::Input(format!(
413            "cannot back up the existing voice {} to {}: {error}",
414            path.display(),
415            backup.display()
416        ))
417    })?;
418    // Write the replacement to a sibling first, then rename over the target: a crash mid-write
419    // must not leave a half-written vector where a valid voice used to be.
420    let staging = path.with_extension("spk.incoming");
421    if staging.exists() {
422        fs::remove_file(&staging).map_err(|error| {
423            FttsError::Input(format!(
424                "cannot clear the stale staging file {}: {error}",
425                staging.display()
426            ))
427        })?;
428    }
429    write_speaker_vector_new(&staging, vector)?;
430    fs::rename(&staging, path).map_err(|error| {
431        FttsError::Input(format!(
432            "cannot replace {} with the new voice: {error}",
433            path.display()
434        ))
435    })?;
436    Ok(backup)
437}
438
439fn decode_speaker_vector(path: &Path, bytes: &[u8]) -> Result<Vec<f32>, FttsError> {
440    let vector: Vec<f32> = bytes
441        .as_chunks::<4>()
442        .0
443        .iter()
444        .map(|quad| f32::from_le_bytes(*quad))
445        .collect();
446    if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
447        return Err(FttsError::Input(format!(
448            "speaker vector {} holds a non-finite value at index {index}; it would poison every \
449             prefill position it is summed into",
450            path.display()
451        )));
452    }
453    Ok(vector)
454}
455
456/// Container formats the embedded decoder does not read; these route through a system decoder,
457/// mirroring how output encoding shells out — synthesis and enrollment themselves never depend
458/// on one.
459const SYSTEM_DECODED_EXTENSIONS: [&str; 6] = ["m4a", "mp3", "aac", "mp4", "ogg", "opus"];
460
461/// A user-owned staging directory for temporary artifacts (transcoded references,
462/// materialized preset voices).
463///
464/// This deliberately avoids the shared system temp dir: on Linux, `/tmp` is world-writable,
465/// the staging names here are predictable (pid + stem), and the files are written by
466/// external decoders that happily follow a pre-planted symlink — so another local user
467/// could truncate an arbitrary victim-writable file, or swap content between our write and
468/// read. A directory under the user's own cache root closes the whole class. Falls back to
469/// the system temp dir only when no home directory exists at all.
470pub(crate) fn private_staging_dir() -> std::io::Result<PathBuf> {
471    #[allow(deprecated)] // un-deprecated in current Rust; the lint fires on older stables
472    let base = std::env::home_dir()
473        .map(|home| home.join(".cache/franken_tts"))
474        .unwrap_or_else(std::env::temp_dir);
475    let dir = base.join("staging");
476    fs::create_dir_all(&dir)?;
477    #[cfg(unix)]
478    {
479        use std::os::unix::fs::PermissionsExt;
480        fs::set_permissions(&dir, fs::Permissions::from_mode(0o700))?;
481    }
482    Ok(dir)
483}
484
485/// Decodes reference audio of any supported container to mono f32 PCM.
486///
487/// WAV and FLAC decode through the embedded pure-Rust path. Compressed containers (m4a, mp3, …)
488/// are first transcoded to a temporary WAV by the first system decoder found — `afconvert` on
489/// macOS, then `ffmpeg` — with a clear error naming both tools when neither exists.
490fn decode_reference_audio_any(path: &Path) -> Result<Vec<f32>, FttsError> {
491    let extension = path
492        .extension()
493        .and_then(|extension| extension.to_str())
494        .map(str::to_ascii_lowercase);
495    let needs_system_decoder = extension
496        .as_deref()
497        .is_some_and(|extension| SYSTEM_DECODED_EXTENSIONS.contains(&extension));
498    if !needs_system_decoder {
499        return decode_reference_audio(path);
500    }
501
502    let staging_dir = private_staging_dir()
503        .map_err(|error| FttsError::Generic(format!("cannot create staging directory: {error}")))?;
504    let staging = staging_dir.join(format!(
505        "ftts-enroll-{}-{}.wav",
506        std::process::id(),
507        path.file_stem()
508            .and_then(|stem| stem.to_str())
509            .unwrap_or("reference")
510    ));
511    let attempts: &[(&str, Vec<&std::ffi::OsStr>)] = &[
512        // Both decoders are told to resample to the speaker encoder's pinned 24 kHz mono here
513        // rather than leaving the source rate intact: phone and Mac voice memos default to
514        // 44.1/48 kHz, and a transcode that preserves them would only move the failure to the
515        // enrollment rate check (frankentts-gra).
516        (
517            "afconvert",
518            vec![
519                "-f".as_ref(),
520                "WAVE".as_ref(),
521                "-d".as_ref(),
522                "LEI16@24000".as_ref(),
523                "-c".as_ref(),
524                "1".as_ref(),
525                path.as_os_str(),
526                staging.as_os_str(),
527            ],
528        ),
529        (
530            "ffmpeg",
531            vec![
532                "-y".as_ref(),
533                "-loglevel".as_ref(),
534                "error".as_ref(),
535                "-i".as_ref(),
536                path.as_os_str(),
537                "-acodec".as_ref(),
538                "pcm_s16le".as_ref(),
539                "-ar".as_ref(),
540                "24000".as_ref(),
541                "-ac".as_ref(),
542                "1".as_ref(),
543                staging.as_os_str(),
544            ],
545        ),
546    ];
547    let mut ran = false;
548    for (tool, arguments) in attempts {
549        match std::process::Command::new(tool).args(arguments).status() {
550            Ok(status) if status.success() => {
551                ran = true;
552                break;
553            }
554            Ok(status) => {
555                let _ = fs::remove_file(&staging);
556                return Err(FttsError::Input(format!(
557                    "{tool} failed decoding reference audio {} (exit {status})",
558                    path.display()
559                )));
560            }
561            Err(_) => continue, // tool not installed; try the next one
562        }
563    }
564    if !ran {
565        return Err(FttsError::Input(format!(
566            "reference audio {} is a compressed container and no system decoder was found; \
567             install afconvert (macOS) or ffmpeg, or supply WAV/FLAC",
568            path.display()
569        )));
570    }
571    let decoded = decode_reference_audio(&staging);
572    let _ = fs::remove_file(&staging);
573    decoded
574}
575
576fn decode_reference_audio(path: &Path) -> Result<Vec<f32>, FttsError> {
577    let file = fs::File::open(path).map_err(|error| {
578        FttsError::Input(format!(
579            "cannot open reference audio {}: {error}",
580            path.display()
581        ))
582    })?;
583    let mut hint = Hint::new();
584    if let Some(extension) = path.extension().and_then(|extension| extension.to_str()) {
585        hint.with_extension(extension);
586    }
587    let stream = MediaSourceStream::new(Box::new(file), Default::default());
588    let probed = get_probe()
589        .format(
590            &hint,
591            stream,
592            &FormatOptions::default(),
593            &MetadataOptions::default(),
594        )
595        .map_err(|error| {
596            FttsError::Input(format!(
597                "cannot identify reference audio {}: {error}",
598                path.display()
599            ))
600        })?;
601    let mut format = probed.format;
602    let track = format.default_track().ok_or_else(|| {
603        FttsError::Input(format!(
604            "reference audio {} has no default audio track",
605            path.display()
606        ))
607    })?;
608    let track_id = track.id;
609    let mut decoder = get_codecs()
610        .make(&track.codec_params, &DecoderOptions::default())
611        .map_err(|error| {
612            FttsError::Input(format!(
613                "cannot decode reference audio {}: {error}",
614                path.display()
615            ))
616        })?;
617    let mut sample_rate = None;
618    let mut mono = Vec::new();
619    loop {
620        let packet = match format.next_packet() {
621            Ok(packet) => packet,
622            Err(SymphoniaError::IoError(error))
623                if error.kind() == std::io::ErrorKind::UnexpectedEof =>
624            {
625                break;
626            }
627            Err(error) => {
628                return Err(FttsError::Input(format!(
629                    "cannot read reference audio {}: {error}",
630                    path.display()
631                )));
632            }
633        };
634        if packet.track_id() != track_id {
635            continue;
636        }
637        let decoded = decoder.decode(&packet).map_err(|error| {
638            FttsError::Input(format!(
639                "cannot decode reference audio {}: {error}",
640                path.display()
641            ))
642        })?;
643        let spec = *decoded.spec();
644        match sample_rate {
645            Some(rate) if rate != spec.rate => {
646                return Err(FttsError::Input(format!(
647                    "reference audio {} changed sample rate mid-stream ({rate} to {} Hz)",
648                    path.display(),
649                    spec.rate
650                )));
651            }
652            None => sample_rate = Some(spec.rate),
653            Some(_) => {}
654        }
655        let channels = spec.channels.count();
656        let mut samples = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
657        samples.copy_interleaved_ref(decoded);
658        for frame in samples.samples().chunks_exact(channels) {
659            mono.push(frame.iter().sum::<f32>() / channels as f32);
660        }
661    }
662    let rate = sample_rate.ok_or_else(|| {
663        FttsError::Input(format!(
664            "reference audio {} contains no decodable samples",
665            path.display()
666        ))
667    })?;
668    if mono.is_empty() {
669        return Err(FttsError::Input(format!(
670            "reference audio {} contains no PCM samples",
671            path.display()
672        )));
673    }
674    let pcm = resample_to_speaker_rate(mono, rate);
675    // Downsampling shortens the signal, and a clip of a few samples at a high source rate can
676    // round to nothing. The mel front end would then see an empty slice, so the emptiness check
677    // has to be made against the PCM actually handed on, not only against what was decoded.
678    if pcm.is_empty() {
679        return Err(FttsError::Input(format!(
680            "reference audio {} is too short to resample from {rate} Hz to \
681             {SPEAKER_SAMPLE_RATE_HZ} Hz; supply a longer recording",
682            path.display()
683        )));
684    }
685    Ok(pcm)
686}
687
688/// Resamples decoded mono PCM to the speaker encoder's pinned rate.
689///
690/// Compressed references already arrive at 24 kHz because the system decoder is told to convert
691/// (`frankentts-gra`), but a `.wav` or `.flac` is read directly and can be any rate — 44.1 and 48
692/// kHz being what every phone, Mac voice memo, and DAW export actually produces. Refusing those
693/// pushed the identical resample onto the user as an `ffmpeg` incantation, so it happens here.
694///
695/// Audio already at the pinned rate is returned untouched, so this cannot perturb any existing
696/// enrollment: it only turns a former hard error into a working path.
697///
698/// Windowed-sinc (Lanczos-3) with the kernel cutoff clamped to the lower of the two rates, which
699/// is what suppresses aliasing on the common downsampling direction. Taps are normalized by their
700/// own sum so DC gain stays 1 even where the window runs off the ends of the signal.
701fn resample_to_speaker_rate(mono: Vec<f32>, from_rate: u32) -> Vec<f32> {
702    if from_rate == SPEAKER_SAMPLE_RATE_HZ {
703        return mono;
704    }
705    resample_lanczos(&mono, from_rate, SPEAKER_SAMPLE_RATE_HZ)
706}
707
708/// The Lanczos-6 core, rate-agnostic: also carries the denoiser's 24 <-> 48 kHz round trip.
709fn resample_lanczos(mono: &[f32], from_rate: u32, to_rate: u32) -> Vec<f32> {
710    if from_rate == to_rate {
711        return mono.to_vec();
712    }
713    // Six lobes, not three: with the cutoff at the output Nyquist a 3-lobe kernel's transition
714    // band sits inside the passband — measured -2.2 dB at 10 kHz for 48->24 kHz and only ~18 dB
715    // of alias rejection, right where the speaker encoder reads sibilance. Six lobes halves the
716    // transition width and pushes rejection past 40 dB for double the (still trivial) tap count.
717    const LOBES: f64 = 6.0;
718    let ratio = f64::from(to_rate) / f64::from(from_rate);
719    let cutoff = ratio.min(1.0);
720    let half = (LOBES / cutoff).ceil() as isize;
721    let out_len = ((mono.len() as f64) * ratio).round() as usize;
722
723    let mut out = Vec::with_capacity(out_len);
724    for index in 0..out_len {
725        let center = index as f64 / ratio;
726        let first = center.floor() as isize - half + 1;
727        let mut acc = 0.0_f64;
728        let mut norm = 0.0_f64;
729        for tap in first..first + 2 * half {
730            if tap < 0 {
731                continue;
732            }
733            let Some(sample) = mono.get(tap as usize) else {
734                break;
735            };
736            let weight = lanczos_tap(center - tap as f64, cutoff, LOBES);
737            acc += weight * f64::from(*sample);
738            norm += weight;
739        }
740        out.push(if norm.abs() > 1e-12 {
741            (acc / norm) as f32
742        } else {
743            0.0
744        });
745    }
746    out
747}
748
749/// STFT window for reference denoising: 512 samples is ~21 ms at the pinned 24 kHz, long enough
750/// to resolve a noise floor between words and short enough not to smear plosives.
751const DENOISE_FRAME: usize = 512;
752
753/// Three-quarter overlap. Hann at hop `N/4` overlap-adds smoothly, which is what keeps gain
754/// changes from becoming audible frame edges.
755const DENOISE_HOP: usize = DENOISE_FRAME / 4;
756
757/// Decision-directed smoothing for the a priori SNR. Ephraim and Malah's 0.98 is calmer but lags
758/// onsets; 0.92 tracks a voice that starts and stops mid-recording.
759const DD_ALPHA: f32 = 0.92;
760
761/// Floor gain, −35 dB. OM-LSA never gates a bin fully closed: leaving a quiet, *stationary* bed
762/// is what stops residual noise from flickering into musical tones.
763///
764/// This is the right knob for "deeper pauses", and the only safe one — it applies where the
765/// presence probability has already decided a bin is noise, so lowering it buys silence between
766/// words without touching a bin the estimator thinks holds voice. Reaching for a more aggressive
767/// *noise estimate* instead is what damages speech.
768const OMLSA_GAIN_FLOOR: f32 = 0.017_782_79;
769
770/// Frames per block over which each bin's noise floor is estimated. ~1.4 s at this hop: long
771/// enough that speech is sparse within a block, short enough to follow room tone that drifts.
772const NOISE_BLOCK_FRAMES: usize = 256;
773
774/// Prior probability that a bin holds no speech, used in the likelihood ratio. Slightly above a
775/// half so that ambiguous bins lean toward suppression rather than passing noise through.
776const SPEECH_ABSENCE_PRIOR: f32 = 0.6;
777
778/// Bias compensation for reading a low quantile of an exponentially distributed power as its
779/// mean: for `Exp(mu)` the q-quantile is `-mu ln(1-q)`, so the 10th percentile UNDERSTATES the
780/// mean 9.49x. Without this factor, pure-pause frames measure a posteriori SNRs of ~7-10 against
781/// the uncorrected floor, the presence estimator reads them as speech, and the gain never
782/// approaches the floor (measured: 2.5 dB of pause reduction instead of ~20). This is the same
783/// role IMCRA's B_min plays for its minimum statistic, recomputed for the quantile used here.
784#[allow(clippy::excessive_precision)]
785const NOISE_QUANTILE_BIAS: f32 = 9.491_221; // 1 / -ln(1 - NOISE_INIT_QUANTILE)
786
787/// Quantile of each bin's power, across the whole recording, taken as its initial noise floor.
788/// Speech is sparse in time, so a low quantile of a bin is the room rather than the voice.
789const NOISE_INIT_QUANTILE: f32 = 0.1;
790
791/// Single-channel speech enhancement: MMSE-LSA gains, decision-directed SNR, OM-LSA presence
792/// weighting, over a noise floor initialised offline and then tracked recursively.
793///
794/// Enrollment noise is not cosmetic: it is encoded into the x-vector and then reproduced in every
795/// utterance the cloned voice speaks (measured — cleaning a real 53 s reference dropped the
796/// synthesized output's pause floor by 19.5 dB). This removes the stationary part of it.
797///
798/// **Why this and not spectral subtraction.** Subtracting an estimated noise magnitude minimises
799/// squared error in the *spectrum*, which is the wrong objective for something a listener judges
800/// and a speaker encoder reads: it punches holes in low-SNR bins, producing musical noise, and it
801/// removes real signal along with the noise (measured here at 33% of peak burst energy before
802/// this replaced it). Three pieces fix that, and they compose:
803///
804/// 1. **MMSE-LSA** (Ephraim & Malah 1985) estimates the *log* amplitude, matching how loudness is
805///    perceived, and yields the gain `ξ/(1+ξ) · exp(½·E₁(ν))`. The exponential-integral term is
806///    what makes it gentle where the a posteriori SNR is uncertain instead of gating hard.
807/// 2. **Decision-directed a priori SNR** (same paper) smooths ξ across frames using the previous
808///    frame's own estimate. This is the specific mechanism that suppresses musical noise: isolated
809///    noise peaks never get a confident ξ, so they are never sharply attenuated *or* passed.
810/// 3. **OM-LSA** (Cohen & Berdugo 2001) blends that gain toward the floor by the speech-presence
811///    probability, `G = G_LSA^p · G_min^(1−p)`, so bins that are probably noise settle to a
812///    constant bed rather than being tracked.
813///
814/// **Why the noise floor is initialised offline rather than by minimum statistics.** IMCRA's
815/// online minimum tracking exists because a streaming denoiser cannot see the future. Enrollment
816/// can: the file is already on disk, so a low quantile of each bin over the whole recording is a
817/// better starting floor than any causal estimator's, with none of the convergence transient.
818/// An earlier revision here did run minimum statistics, and it is instructive why that was
819/// removed rather than debugged: seeded from frame 0 of a reference that opens on speech, the
820/// refined minimum locked above the speech level, which drove the presence probability to zero,
821/// which unfroze the noise update, which let the noise estimate absorb the voice — a positive
822/// feedback that left ~10% of every burst after the first. Speech presence here instead comes
823/// from the likelihood ratio in ξ and ν, which is self-correcting: it cannot conclude "no speech"
824/// about a bin whose own a priori SNR is high.
825///
826/// Nonstationary noise is still tracked, by the recursive average that the presence probability
827/// gates — the offline quantile only sets where that average starts.
828///
829/// This is the state of the art among methods that need no trained weights. Neural enhancers
830/// (DeepFilterNet and friends) do beat it, at the cost of shipping and running another model —
831/// which is not a trade this CLI should make silently for an enrollment preprocessing step.
832///
833/// Deliberately conservative even so: the speaker encoder reads breath, sibilance, and room as
834/// part of identity, so this stays opt-in (`--denoise`) per the project's doctrine that a lever
835/// which can damage speaker identity ships behind a named switch until blind listening clears it.
836///
837/// Phase is preserved untouched; only per-bin magnitude is scaled.
838/// Where `ftts pull` lands the neural denoiser, relative to the model root.
839pub const DENOISE_ARTIFACT_RELPATH: &str = "denoise/fastenhancer-s-48k.safetensors";
840
841/// The FastEnhancer denoise path: 24 kHz reference up to the model's native 48 kHz,
842/// through the ported network, back down to 24 kHz.
843///
844/// Returns `Ok(None)` when the neural route is unavailable (artifact not pulled) or the
845/// user forced the classic engine with `FTTS_DENOISE_ENGINE=omlsa` — the caller falls back
846/// to the spectral-subtraction reference. A *present but malformed* artifact is an error,
847/// not a silent fallback: repairing it is one `ftts pull --force` away, and quietly handing
848/// the user a different denoiser than the one they pulled would misreport what cleaned
849/// their reference.
850///
851/// The network runs at 48 kHz because that is what its checkpoint was trained on (with
852/// low-pass augmentation down to 24 kHz content, so band-limited input is in-distribution).
853/// The round trip uses the same Lanczos-6 resampler enrollment already trusts, and the
854/// input is zero-padded to the model's hop grid so the STFT round trip returns every
855/// sample, then trimmed back to the original length.
856fn neural_denoise_reference(
857    bundle: &ModelBundle,
858    pcm24k: &[f32],
859) -> Result<Option<Vec<f32>>, FttsError> {
860    let path = bundle.root.join(DENOISE_ARTIFACT_RELPATH);
861    if !path.is_file() {
862        return Ok(None);
863    }
864    if std::env::var("FTTS_DENOISE_ENGINE").is_ok_and(|v| v.eq_ignore_ascii_case("omlsa")) {
865        return Ok(None);
866    }
867    let enhancer = ftts_artifacts::enhance_loader::open_enhancer(&path).map_err(|error| {
868        FttsError::ArtifactFormat(format!(
869            "denoiser artifact {} is unreadable ({error}); re-fetch it with `ftts pull --force`",
870            path.display()
871        ))
872    })?;
873    Ok(Some(enhancer.enhance_24k(pcm24k)))
874}
875
876fn denoise_reference(pcm: &[f32]) -> Vec<f32> {
877    // A floor estimated from a handful of frames is just the clip's own spectrum; below ~a
878    // quarter second there is nothing honest to subtract, so the clip passes through untouched
879    // (a single-frame "estimate" measured as flattening the whole clip toward the gain floor).
880    const DENOISE_MIN_FRAMES: usize = 32;
881    if pcm.len() < DENOISE_FRAME + (DENOISE_MIN_FRAMES - 1) * DENOISE_HOP {
882        return pcm.to_vec();
883    }
884    let mut planner = rustfft::FftPlanner::<f32>::new();
885    let forward = planner.plan_fft_forward(DENOISE_FRAME);
886    let inverse = planner.plan_fft_inverse(DENOISE_FRAME);
887
888    let window: Vec<f32> = (0..DENOISE_FRAME)
889        .map(|n| {
890            let phase = std::f32::consts::TAU * n as f32 / DENOISE_FRAME as f32;
891            0.5 - 0.5 * phase.cos()
892        })
893        .collect();
894
895    let bins = DENOISE_FRAME / 2 + 1;
896    let starts: Vec<usize> = (0..=pcm.len() - DENOISE_FRAME)
897        .step_by(DENOISE_HOP)
898        .collect();
899
900    // Pass 1: every frame's power spectrum. Only the magnitudes are kept; retaining the complex
901    // frames would save the second FFT at 8× the memory, which a several-minute reference feels.
902    let mut powers: Vec<Vec<f32>> = Vec::with_capacity(starts.len());
903    let mut scratch: Vec<rustfft::num_complex::Complex<f32>> =
904        vec![rustfft::num_complex::Complex::new(0.0, 0.0); DENOISE_FRAME];
905    for &start in &starts {
906        for (slot, n) in scratch.iter_mut().zip(0..DENOISE_FRAME) {
907            *slot = rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0);
908        }
909        forward.process(&mut scratch);
910        powers.push((0..bins).map(|bin| scratch[bin].norm_sqr()).collect());
911    }
912
913    // Noise floor per bin: the MINIMUM over blocks of a low within-block quantile.
914    //
915    // There is deliberately no feedback here. A recursive noise average has to be gated by a
916    // speech-presence estimate, which in turn divides by the noise — and any error in that loop
917    // compounds: one speech frame admitted into the floor raises it, which lowers the presence
918    // estimate, which admits more speech. Both earlier revisions of this function died that way.
919    // Reading the whole file at once removes the loop rather than tuning it.
920    //
921    // The min-over-blocks reduction is load-bearing: a bare per-block quantile assumes speech is
922    // sparse WITHIN every 1.4 s block, and a bin that stays voiced across one whole block (a held
923    // vowel, a low harmonic mid-sentence) would have its own signal adopted as that block's floor
924    // and be gated to the floor gain — measured at ~28 dB of deletion on a sustained tone. Taking
925    // the minimum across blocks only requires the bin to be quiet somewhere in the recording,
926    // which is what "noise floor" actually means.
927    let blocks = powers.len().div_ceil(NOISE_BLOCK_FRAMES);
928    let mut noise = vec![f32::INFINITY; bins];
929    let mut column: Vec<f32> = Vec::with_capacity(NOISE_BLOCK_FRAMES);
930    for block in 0..blocks {
931        let span = block * NOISE_BLOCK_FRAMES..((block + 1) * NOISE_BLOCK_FRAMES).min(powers.len());
932        for (bin, slot) in noise.iter_mut().enumerate() {
933            column.clear();
934            column.extend(powers[span.clone()].iter().map(|frame| frame[bin]));
935            column.sort_by(f32::total_cmp);
936            let rank = ((column.len() as f32 - 1.0) * NOISE_INIT_QUANTILE).round() as usize;
937            *slot = slot.min(column[rank].max(1e-12) * NOISE_QUANTILE_BIAS);
938        }
939    }
940
941    // Decision-directed state: last frame's gain and a posteriori SNR, per bin.
942    let mut prev_gain = vec![1.0_f32; bins];
943    let mut prev_gamma = vec![1.0_f32; bins];
944
945    let mut out = vec![0.0_f32; pcm.len()];
946    let mut weight = vec![0.0_f32; pcm.len()];
947
948    for (index, &start) in starts.iter().enumerate() {
949        let mut frame: Vec<rustfft::num_complex::Complex<f32>> = (0..DENOISE_FRAME)
950            .map(|n| rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0))
951            .collect();
952        forward.process(&mut frame);
953        let power = &powers[index];
954
955        for bin in 0..bins {
956            let gamma = (power[bin] / noise[bin]).min(1e6);
957            let xi = (DD_ALPHA * prev_gain[bin].powi(2) * prev_gamma[bin]
958                + (1.0 - DD_ALPHA) * (gamma - 1.0).max(0.0))
959            .max(1e-6);
960
961            let nu = (xi / (1.0 + xi)) * gamma;
962            let lsa =
963                ((xi / (1.0 + xi)) * (0.5 * exponential_integral_e1(nu)).exp()).clamp(0.0, 1.0);
964
965            // Speech-presence probability by the likelihood ratio (Ephraim & Malah's signal
966            // presence uncertainty). High ξ with a matching ν drives this to 1, so a bin that is
967            // plainly speech can never be talked into being noise.
968            let odds = SPEECH_ABSENCE_PRIOR / (1.0 - SPEECH_ABSENCE_PRIOR);
969            let presence = 1.0 / (1.0 + odds * (1.0 + xi) * (-nu).exp());
970            let presence = presence.clamp(0.0, 1.0);
971
972            let gain = (lsa.max(OMLSA_GAIN_FLOOR).powf(presence)
973                * OMLSA_GAIN_FLOOR.powf(1.0 - presence))
974            .clamp(OMLSA_GAIN_FLOOR, 1.0);
975
976            prev_gain[bin] = gain;
977            prev_gamma[bin] = gamma;
978
979            frame[bin] *= gain;
980            let mirror = DENOISE_FRAME - bin;
981            // DC (bin 0) has no mirror, and Nyquist (bin N/2) *is* its own mirror — scaling it
982            // through this branch as well would apply `gain` twice there.
983            if mirror != bin && mirror < DENOISE_FRAME {
984                frame[mirror] *= gain;
985            }
986        }
987
988        inverse.process(&mut frame);
989        let scale = 1.0 / DENOISE_FRAME as f32;
990        for n in 0..DENOISE_FRAME {
991            out[start + n] += frame[n].re * scale * window[n];
992            weight[start + n] += window[n] * window[n];
993        }
994    }
995
996    // A sample under-covered by the window stack cannot be normalized honestly: near the edges
997    // out[n] is dominated by circular-convolution leakage from the rest of the frame, and
998    // dividing that by a window energy as small as ~2e-6 manufactures a spike (measured 1.5x
999    // input peak with a bare non-zero guard). Anything below a tenth of the steady-state COLA
1000    // sum (1.5 for periodic Hann at hop N/4) keeps the original PCM instead.
1001    const WOLA_MIN_WEIGHT: f32 = 0.15;
1002    for (sample, energy) in out.iter_mut().zip(weight.iter()) {
1003        if *energy > WOLA_MIN_WEIGHT {
1004            *sample /= *energy;
1005        }
1006    }
1007    // The head and tail lie outside any fully-stacked window and keep their original samples
1008    // rather than a partially-normalized reconstruction.
1009    let covered =
1010        starts.first().copied().unwrap_or(0)..starts.last().map_or(0, |last| last + DENOISE_FRAME);
1011    for (index, sample) in out.iter_mut().enumerate() {
1012        if !covered.contains(&index) || weight[index] <= WOLA_MIN_WEIGHT {
1013            *sample = pcm[index];
1014        }
1015    }
1016    out
1017}
1018
1019/// The exponential integral `E₁(x) = ∫ₓ^∞ e^{−t}/t dt`, for `x > 0`.
1020///
1021/// This is the term that makes MMSE-LSA gentle rather than gating: it grows without bound as the
1022/// a priori SNR falls, so the log-amplitude estimate backs off smoothly instead of snapping shut.
1023/// Abramowitz & Stegun 5.1.53 below 1 and 5.1.56 above it; both are accurate to ~2e-7, far inside
1024/// what a spectral gain needs.
1025fn exponential_integral_e1(x: f32) -> f32 {
1026    if x <= 0.0 {
1027        // ν ≤ 0 cannot arise from a non-negative SNR, but a denormal would otherwise return NaN
1028        // and poison the frame.
1029        return 0.0;
1030    }
1031    let x = f64::from(x);
1032    let value = if x < 1.0 {
1033        // A&S 5.1.53: E₁(x) + ln x = polynomial in x.
1034        const A: [f64; 6] = [
1035            -0.577_215_664_9,
1036            0.999_991_93,
1037            -0.249_910_55,
1038            0.055_199_68,
1039            -0.009_760_04,
1040            0.001_078_57,
1041        ];
1042        let mut acc = 0.0;
1043        for (power, coefficient) in A.iter().enumerate() {
1044            acc += coefficient * x.powi(power as i32);
1045        }
1046        acc - x.ln()
1047    } else {
1048        // A&S 5.1.56: x·e^x·E₁(x) = rational in x.
1049        const A: [f64; 4] = [8.573_328_74, 18.059_016_97, 8.634_760_89, 0.267_773_734];
1050        const B: [f64; 4] = [9.573_322_34, 25.632_956_15, 21.099_653_08, 3.958_496_93];
1051        let numerator = x.powi(4) + A[0] * x.powi(3) + A[1] * x * x + A[2] * x + A[3];
1052        let denominator = x.powi(4) + B[0] * x.powi(3) + B[1] * x * x + B[2] * x + B[3];
1053        (numerator / denominator) / (x * x.exp())
1054    };
1055    value as f32
1056}
1057
1058/// Dereverberation runs its own STFT, deliberately coarser in time than the denoiser's.
1059///
1060/// The two want opposite things. Denoising wants short frames so a gain change lands inside a
1061/// phoneme; prediction wants each frame to cover enough of the room's tail that a tractable
1062/// number of taps can span it. At a 5.3 ms hop, a 24-tap filter reaches 128 ms — against an
1063/// 810 ms reverb that removed 0.01 s of RT60, i.e. nothing (measured). A 10.7 ms hop with 40 taps
1064/// reaches ~427 ms, which is the fraction of the tail single-channel prediction can model without
1065/// the covariance becoming both enormous and ill-conditioned.
1066const DEREVERB_FRAME: usize = 1024;
1067const DEREVERB_HOP: usize = 256;
1068
1069/// Prediction taps: ~427 ms of tail at [`DEREVERB_HOP`].
1070const DEREVERB_TAPS: usize = 40;
1071
1072/// Frames skipped before prediction starts, so the direct sound and its early reflections are
1073/// never predictable from the regressor and therefore never subtracted. This delay is the whole
1074/// reason WPE dereverberates instead of just whitening the voice.
1075const DEREVERB_DELAY: usize = 2;
1076
1077/// Alternations between "estimate the speech variance" and "re-fit the filter". The variance
1078/// estimate is what makes the fit ignore loud speech frames and key on the tail; two passes are
1079/// enough to converge in practice, three leaves margin.
1080const DEREVERB_ITERATIONS: usize = 3;
1081
1082/// Diagonal loading on the covariance, relative to its own trace. Silent bins are rank-deficient
1083/// and would otherwise produce an arbitrary filter that injects noise instead of removing tail.
1084const DEREVERB_LOADING: f64 = 1e-4;
1085
1086/// What a `--dereverb` enrollment measured, so the CLI reports the effect rather than asserting it.
1087#[derive(Clone, Copy, Debug)]
1088pub struct DereverbReport {
1089    /// Reverberation time equivalent of the reference, before dereverberation.
1090    pub before_rt60_s: f32,
1091    /// The same measure afterwards.
1092    pub after_rt60_s: f32,
1093}
1094
1095/// Blind single-channel dereverberation by Weighted Prediction Error (Nakatani et al., 2010).
1096///
1097/// # Why this is a separate lever from `--denoise`
1098///
1099/// Reverb is *convolutive*: the microphone hears the voice convolved with the room's impulse
1100/// response. Denoising subtracts an *additive* stationary floor. The two do not overlap at all,
1101/// which is why running the denoiser on a reverberant reference moves the noise floor by 0.0 dB
1102/// and leaves the wetness untouched — measured, on exactly the recording that prompted this.
1103///
1104/// # Why it matters for enrollment specifically
1105///
1106/// The speaker encoder cannot separate voice from room, so a wet reference enrolls the room as
1107/// part of the speaker's identity and every utterance the clone speaks is rendered in that room
1108/// (measured: a 0.81 s reference produced a 0.79 s clone; a 0.66 s reference produced 0.68 s).
1109/// Drying the reference is therefore not cosmetic — it changes who the model thinks it is
1110/// imitating.
1111///
1112/// # The method
1113///
1114/// Late reverberation at frame `t` is, by construction, a linear function of the *past* of the
1115/// same signal: it is what earlier sound has decayed into. So per frequency bin, fit a linear
1116/// predictor from frames `t-D-L+1 ..= t-D` and subtract what it predicts. The delay `D` is what
1117/// protects the direct path: the speech itself is not predictable at that lag, the room's tail is.
1118///
1119/// The weighting is the "WPE" part and the reason it beats plain linear prediction. Each frame is
1120/// divided by the current estimate of the speech power there, so loud vowels — where the residual
1121/// is dominated by speech, not tail — stop dominating the fit. Estimating that power needs the
1122/// dereverberated signal, which needs the filter, so the two alternate for a few iterations.
1123///
1124/// Only late reverberation is removed. Early reflections arrive inside the protected delay by
1125/// design, so a very close, very live room is improved less than a distant one.
1126fn dereverb_reference(pcm: &[f32]) -> Vec<f32> {
1127    if pcm.len() < DEREVERB_FRAME * 4 {
1128        return pcm.to_vec();
1129    }
1130    let mut planner = rustfft::FftPlanner::<f32>::new();
1131    let forward = planner.plan_fft_forward(DEREVERB_FRAME);
1132    let inverse = planner.plan_fft_inverse(DEREVERB_FRAME);
1133
1134    let window: Vec<f32> = (0..DEREVERB_FRAME)
1135        .map(|n| {
1136            let phase = std::f32::consts::TAU * n as f32 / DEREVERB_FRAME as f32;
1137            0.5 - 0.5 * phase.cos()
1138        })
1139        .collect();
1140
1141    let bins = DEREVERB_FRAME / 2 + 1;
1142    let starts: Vec<usize> = (0..=pcm.len() - DEREVERB_FRAME)
1143        .step_by(DEREVERB_HOP)
1144        .collect();
1145    let frames = starts.len();
1146    if frames <= DEREVERB_DELAY + DEREVERB_TAPS + 2 {
1147        return pcm.to_vec();
1148    }
1149
1150    // Observed spectra, kept complex: prediction needs phase, unlike the magnitude-only denoiser.
1151    let mut observed: Vec<Vec<Complex64>> = Vec::with_capacity(frames);
1152    let mut scratch: Vec<rustfft::num_complex::Complex<f32>> =
1153        vec![rustfft::num_complex::Complex::new(0.0, 0.0); DEREVERB_FRAME];
1154    for &start in &starts {
1155        for (slot, n) in scratch.iter_mut().zip(0..DEREVERB_FRAME) {
1156            *slot = rustfft::num_complex::Complex::new(pcm[start + n] * window[n], 0.0);
1157        }
1158        forward.process(&mut scratch);
1159        observed.push(
1160            scratch[..bins]
1161                .iter()
1162                .map(|value| Complex64::new(f64::from(value.re), f64::from(value.im)))
1163                .collect(),
1164        );
1165    }
1166
1167    let mut desired = observed.clone();
1168    for _ in 0..DEREVERB_ITERATIONS {
1169        for bin in 0..bins {
1170            // Speech power per frame, from the current estimate. The floor keeps a silent frame
1171            // from receiving unbounded weight and hijacking the fit.
1172            let mut power: Vec<f64> = (0..frames).map(|t| desired[t][bin].norm_sqr()).collect();
1173            let mean = power.iter().sum::<f64>() / frames as f64;
1174            let floor = (mean * 1e-6).max(1e-12);
1175            for value in &mut power {
1176                *value = value.max(floor);
1177            }
1178
1179            let taps = DEREVERB_TAPS;
1180            let mut covariance = vec![Complex64::new(0.0, 0.0); taps * taps];
1181            let mut cross = vec![Complex64::new(0.0, 0.0); taps];
1182            for t in (DEREVERB_DELAY + taps)..frames {
1183                let weight = 1.0 / power[t];
1184                // Regressor: the observed signal at increasing lag past the protected delay.
1185                let regressor: Vec<Complex64> = (0..taps)
1186                    .map(|lag| observed[t - DEREVERB_DELAY - lag][bin])
1187                    .collect();
1188                for row in 0..taps {
1189                    let scaled = regressor[row] * weight;
1190                    for column in row..taps {
1191                        covariance[row * taps + column] += scaled * regressor[column].conj();
1192                    }
1193                    cross[row] += scaled * observed[t][bin].conj();
1194                }
1195            }
1196            // Hermitian: fill the lower triangle from the upper one that was accumulated.
1197            for row in 0..taps {
1198                for column in 0..row {
1199                    covariance[row * taps + column] = covariance[column * taps + row].conj();
1200                }
1201            }
1202            let trace: f64 = (0..taps).map(|i| covariance[i * taps + i].re).sum();
1203            if trace <= 0.0 {
1204                continue;
1205            }
1206            let loading = trace / taps as f64 * DEREVERB_LOADING;
1207            for i in 0..taps {
1208                covariance[i * taps + i] += Complex64::new(loading, 0.0);
1209            }
1210
1211            let Some(filter) = solve_complex_system(&mut covariance, &mut cross, taps) else {
1212                continue;
1213            };
1214            for t in 0..frames {
1215                if t < DEREVERB_DELAY + taps {
1216                    desired[t][bin] = observed[t][bin];
1217                    continue;
1218                }
1219                let mut tail = Complex64::new(0.0, 0.0);
1220                for (lag, coefficient) in filter.iter().enumerate() {
1221                    tail += coefficient.conj() * observed[t - DEREVERB_DELAY - lag][bin];
1222                }
1223                desired[t][bin] = observed[t][bin] - tail;
1224            }
1225        }
1226    }
1227
1228    // Overlap-add the dereverberated spectra back, mirroring the conjugate half so the inverse
1229    // transform yields a real signal.
1230    let mut out = vec![0.0_f32; pcm.len()];
1231    let mut weight = vec![0.0_f32; pcm.len()];
1232    for (index, &start) in starts.iter().enumerate() {
1233        let mut frame = vec![rustfft::num_complex::Complex::new(0.0_f32, 0.0); DEREVERB_FRAME];
1234        for bin in 0..bins {
1235            let value = desired[index][bin];
1236            #[allow(clippy::cast_possible_truncation)]
1237            let value = rustfft::num_complex::Complex::new(value.re as f32, value.im as f32);
1238            frame[bin] = value;
1239            let mirror = DEREVERB_FRAME - bin;
1240            if mirror != bin && mirror < DEREVERB_FRAME {
1241                frame[mirror] = value.conj();
1242            }
1243        }
1244        inverse.process(&mut frame);
1245        let scale = 1.0 / DEREVERB_FRAME as f32;
1246        for n in 0..DEREVERB_FRAME {
1247            out[start + n] += frame[n].re * scale * window[n];
1248            weight[start + n] += window[n] * window[n];
1249        }
1250    }
1251    for (sample, energy) in out.iter_mut().zip(weight.iter()) {
1252        if *energy > 1e-6 {
1253            *sample /= *energy;
1254        }
1255    }
1256    let covered =
1257        starts.first().copied().unwrap_or(0)..starts.last().map_or(0, |last| last + DEREVERB_FRAME);
1258    for (index, sample) in out.iter_mut().enumerate() {
1259        if !covered.contains(&index) || weight[index] <= 1e-6 {
1260            *sample = pcm[index];
1261        }
1262    }
1263    out
1264}
1265
1266/// Double-precision complex scalar for the normal equations.
1267///
1268/// The covariance is accumulated over thousands of frames and then inverted; doing that in f32
1269/// loses conditioning on quiet bins, which is where a bad filter does the most audible damage.
1270type Complex64 = rustfft::num_complex::Complex<f64>;
1271
1272/// Solves `a x = b` by Gaussian elimination with partial pivoting, consuming both.
1273///
1274/// Returns `None` when the system is singular to working precision, which the caller treats as
1275/// "leave this bin alone" rather than as a failure — a bin with no energy has no tail to remove.
1276fn solve_complex_system(
1277    a: &mut [Complex64],
1278    b: &mut [Complex64],
1279    n: usize,
1280) -> Option<Vec<Complex64>> {
1281    for column in 0..n {
1282        let (pivot, magnitude) = (column..n).fold((column, 0.0_f64), |best, row| {
1283            let candidate = a[row * n + column].norm_sqr();
1284            if candidate > best.1 {
1285                (row, candidate)
1286            } else {
1287                best
1288            }
1289        });
1290        if magnitude <= f64::MIN_POSITIVE {
1291            return None;
1292        }
1293        if pivot != column {
1294            for k in 0..n {
1295                a.swap(pivot * n + k, column * n + k);
1296            }
1297            b.swap(pivot, column);
1298        }
1299        let diagonal = a[column * n + column];
1300        for row in (column + 1)..n {
1301            let factor = a[row * n + column] / diagonal;
1302            if factor == Complex64::new(0.0, 0.0) {
1303                continue;
1304            }
1305            for k in column..n {
1306                let value = a[column * n + k] * factor;
1307                a[row * n + k] -= value;
1308            }
1309            let value = b[column] * factor;
1310            b[row] -= value;
1311        }
1312    }
1313    let mut solution = vec![Complex64::new(0.0, 0.0); n];
1314    for row in (0..n).rev() {
1315        let mut accumulator = b[row];
1316        for k in (row + 1)..n {
1317            accumulator -= a[row * n + k] * solution[k];
1318        }
1319        solution[row] = accumulator / a[row * n + row];
1320    }
1321    Some(solution)
1322}
1323
1324/// Reverberation time equivalent, in seconds, from the decay following speech offsets.
1325///
1326/// Reverb leaves no trace in a noise floor — what it does is stretch the energy envelope after
1327/// every stop. Measuring the median decay slope across offsets is what separates "the room rings"
1328/// from "the microphone hisses", two problems whose fixes have nothing in common. Returns `None`
1329/// when the audio has no clear offsets to measure.
1330fn reverb_time_s(pcm: &[f32]) -> Option<f32> {
1331    let hop = (SPEAKER_SAMPLE_RATE_HZ as usize) / 100; // 10 ms
1332    if pcm.len() < hop * 32 {
1333        return None;
1334    }
1335    let envelope: Vec<f32> = pcm
1336        .chunks_exact(hop)
1337        .map(|chunk| {
1338            let energy = chunk.iter().map(|s| s * s).sum::<f32>() / chunk.len() as f32;
1339            10.0 * (energy + 1e-9).log10()
1340        })
1341        .collect();
1342    let peak = envelope.iter().copied().fold(f32::NEG_INFINITY, f32::max);
1343    let span = 15_usize; // 150 ms of decay
1344    let mut slopes: Vec<f32> = Vec::new();
1345    for index in 1..envelope.len().saturating_sub(span) {
1346        if envelope[index] < peak - 25.0 || envelope[index] <= envelope[index - 1] {
1347            continue;
1348        }
1349        let drop = envelope[index] - envelope[index + span - 1];
1350        if drop < 6.0 {
1351            continue;
1352        }
1353        slopes.push(drop / (span as f32 * 0.01));
1354    }
1355    if slopes.is_empty() {
1356        return None;
1357    }
1358    slopes.sort_by(f32::total_cmp);
1359    let median = slopes[slopes.len() / 2];
1360    (median > 0.0).then(|| 60.0 / median)
1361}
1362
1363/// Root-mean-square of the quietest decile of 50 ms windows, in dBFS.
1364///
1365/// Whole-signal RMS hides pause noise behind the speech that dominates it, so enrollment
1366/// diagnostics report the floor between words — the part a denoise actually moves.
1367fn pause_floor_dbfs(pcm: &[f32]) -> f32 {
1368    let span = (SPEAKER_SAMPLE_RATE_HZ as usize) / 20;
1369    if pcm.len() < span {
1370        return f32::NEG_INFINITY;
1371    }
1372    let mut windows: Vec<f32> = pcm
1373        .chunks_exact(span)
1374        .map(|chunk| {
1375            (chunk
1376                .iter()
1377                .map(|s| f64::from(*s) * f64::from(*s))
1378                .sum::<f64>()
1379                / chunk.len() as f64)
1380                .sqrt() as f32
1381        })
1382        .filter(|rms| *rms > 0.0)
1383        .collect();
1384    if windows.is_empty() {
1385        return f32::NEG_INFINITY;
1386    }
1387    windows.sort_by(f32::total_cmp);
1388    let keep = (windows.len() / 10).max(1);
1389    let mean = windows[..keep].iter().sum::<f32>() / keep as f32;
1390    20.0 * mean.log10()
1391}
1392
1393/// One Lanczos tap: a sinc lowpass at `cutoff`, windowed by a wider sinc over `lobes`.
1394fn lanczos_tap(offset: f64, cutoff: f64, lobes: f64) -> f64 {
1395    let scaled = cutoff * offset;
1396    if scaled.abs() >= lobes {
1397        return 0.0;
1398    }
1399    sinc(scaled) * sinc(scaled / lobes)
1400}
1401
1402/// Normalized sinc, `sin(pi x) / (pi x)`, with the removable singularity at zero filled in.
1403fn sinc(x: f64) -> f64 {
1404    if x.abs() < 1e-12 {
1405        return 1.0;
1406    }
1407    let scaled = std::f64::consts::PI * x;
1408    scaled.sin() / scaled
1409}
1410
1411/// Hands the engine a `PreparedText` that was computed before the weights were borrowed.
1412struct PreparedPassThrough {
1413    prepared: PreparedText,
1414}
1415
1416impl TextPreparer for PreparedPassThrough {
1417    fn prepare(
1418        &self,
1419        _text: &str,
1420        _options: &NormalizationOptions,
1421    ) -> Result<PreparedText, TextPreparationError> {
1422        Ok(PreparedText::new(
1423            self.prepared.token_ids.clone(),
1424            NormalizationTrace {
1425                mode: self.prepared.normalization_trace.mode,
1426                unicode_version: self.prepared.normalization_trace.unicode_version.clone(),
1427                changes: self.prepared.normalization_trace.changes.clone(),
1428            },
1429        ))
1430    }
1431}
1432
1433/// A completed synthesis: the codes the talker produced and the audio they decode to.
1434pub struct SynthesizedAudio {
1435    /// Codec frames generated before the stop.
1436    pub frames: u64,
1437    /// Token ids that entered the model path, including the assistant wrapper.
1438    pub prepared_token_count: usize,
1439    /// Mono 24 kHz samples in `[-1, 1]`.
1440    pub pcm: Vec<f32>,
1441    /// Time from synthesis start (prompt work + prefill + first frames) to the first decoded
1442    /// packet of PCM existing. `None` when the run produced no audio. Time-to-first-audio and
1443    /// real-time factor are different products (doctrine: report them separately); this is the
1444    /// TTFA half, excluding model load, which the `load` stage event already bounds.
1445    pub ttfa: Option<std::time::Duration>,
1446}
1447
1448/// Run one utterance end to end: text, codes, PCM.
1449///
1450/// # Errors
1451///
1452/// Engine refusals (admission, budget, cancellation) and model refusals are mapped to their CLI
1453/// exit classes; a zero-frame generation is reported rather than written out as an empty file.
1454#[allow(clippy::too_many_arguments)]
1455pub fn synthesize(
1456    model: &LoadedModel,
1457    engine: &TtsEngine,
1458    request: &SynthesisRequest,
1459    speaker: &[f32],
1460    seed: u64,
1461    cancellation: &CancellationToken,
1462    observer: &dyn SynthesisObserver,
1463) -> Result<SynthesizedAudio, FttsError> {
1464    // 1. Text, once — see the module docs on ordering.
1465    let prepared_raw = model
1466        .tokenizer
1467        .prepare(&request.text, &request.normalization_options)
1468        .map_err(|error| FttsError::Input(format!("text preparation failed: {error}")))?;
1469    let wrapped = TalkerCheckpoint::wrap_target_ids(&prepared_raw.token_ids);
1470    let prepared = PreparedText::new(wrapped.clone(), prepared_raw.normalization_trace);
1471
1472    // 2. The cold-embedding rows this utterance can reach, and nothing else.
1473    let ids = TalkerCheckpoint::utterance_text_ids(&wrapped);
1474    let table = model
1475        .talker
1476        .gather_text_rows(&ids)
1477        .map_err(checkpoint_error)?;
1478
1479    // 3. The prompt header, derived from checkpoint tensors and the caller's speaker vector.
1480    let header = model
1481        .talker
1482        .xvector_header(&table, speaker, CODEC_LANGUAGE_ENGLISH_ID)
1483        .map_err(checkpoint_error)?;
1484    let tts_eos = model.talker.tts_eos(&table);
1485
1486    // 4. Borrowed weights for the generator.
1487    let talker_layers = model.talker.talker_layer_weights();
1488    let micro_layers = model.talker.microdecoder_layer_weights();
1489    let residual = model.talker.residual_embedding_slices();
1490    let heads = model.talker.microdecoder_head_slices();
1491    // The microdecoder's internal tables cover depths 2..=15: the first fourteen of the same
1492    // fifteen-table set the talker feedback path uses.
1493    let micro_residual = &residual[..residual.len() - 1];
1494
1495    let mut generator = QwenGenerator::new_with_artifact(
1496        QwenGeneratorConfig {
1497            talker_config: TalkerConfig::default(),
1498            talker_weights: model.talker.talker_weights(&talker_layers),
1499            text: model.talker.text_weights(&table),
1500            feedback: model.talker.feedback_tables(&residual),
1501            microdecoder_config: MicrodecoderConfig::default(),
1502            microdecoder_weights: model.talker.microdecoder_weights(
1503                &micro_layers,
1504                micro_residual,
1505                &heads,
1506            ),
1507            prompt_mode: PromptMode {
1508                clone_mode: CloneMode::XVector,
1509                non_streaming_mode: false,
1510            },
1511            header,
1512            tts_eos,
1513            reference: None,
1514            // The PRODUCT samples, exactly as the pinned upstream runtime does
1515            // (generation_config.json: do_sample=true, T=0.9, top_k=50, repetition_penalty=1.05,
1516            // subtalker likewise); canonical greedy remains the conformance decoder only. The p7r
1517            // forensics that certified this path: our talker draw stack matched torch's choices
1518            // code-for-code for seven straight frames from the same prefill, the silence defect was
1519            // the subtalker being forced greedy under a sampled talker (a measured silence
1520            // attractor the reference reproduces in that mismatched configuration), and with the
1521            // subtalker sampling per depth the engine's utterance envelope matches the reference's
1522            // sampled runs (peak frame RMS 0.086 with trailing silence). Determinism scope: build +
1523            // ISA + sampler version + seed, 16 draws per frame.
1524            sampling_mode: SamplingMode::Production,
1525            seed,
1526        },
1527        model.artifact.as_deref(),
1528    );
1529
1530    // 5. The engine owns admission, the budget, cancellation, and the frame loop — and the
1531    // codec decodes IN PARALLEL with it: a tee on the generator feeds every produced frame
1532    // through a bounded channel to a scoped codec worker driving the streaming decoder.
1533    // Streamed output is bit-identical to offline decode under every packet schedule (the
1534    // standing streaming==batch gate), so this overlap changes wall time and nothing else.
1535    // Deadlock shape: the worker only ever blocks on `recv` (it always drains), and the
1536    // generator only ever blocks on `send` when the worker is more than 256 frames behind —
1537    // bounded, and covered by the engine's rolling frame budget if the worker wedges.
1538    let preparer = PreparedPassThrough { prepared };
1539    let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<ftts_core::CodeFrame>(256);
1540    let codec = &model.codec;
1541    let synthesis_started = std::time::Instant::now();
1542    let (result, pcm, ttfa) = std::thread::scope(
1543        |scope| -> Result<
1544            (
1545                ftts_core::SynthesisResult,
1546                Vec<f32>,
1547                Option<std::time::Duration>,
1548            ),
1549            FttsError,
1550        > {
1551            let worker = scope.spawn(
1552                move || -> Result<(Vec<f32>, Option<std::time::Duration>), FttsError> {
1553                    // Overlap for real: this thread's int8 ops run serially on a spare core
1554                    // instead of contending for the generator's worker team.
1555                    ftts_kernels::team::bypass_team_on_this_thread();
1556                    const PACKET_FRAMES: usize = 4;
1557                    let mut state = codec.stream_state();
1558                    let mut pcm = Vec::new();
1559                    // `stream_push` REPLACES its output buffer with one packet's samples (see the
1560                    // streaming==offline test), so packets decode into a scratch and append here.
1561                    let mut packet_pcm = Vec::new();
1562                    let mut packet: Vec<i32> = Vec::with_capacity(16 * PACKET_FRAMES);
1563                    let mut packet_frames = 0_usize;
1564                    let mut first_audio_at: Option<std::time::Duration> = None;
1565                    while let Ok(frame) = frame_rx.recv() {
1566                        if frame.codes.len() != 16 {
1567                            return Err(FttsError::Generic(format!(
1568                                "generated frame carries {} codes, expected 16",
1569                                frame.codes.len()
1570                            )));
1571                        }
1572                        for code in &frame.codes {
1573                            packet.push(i32::try_from(*code).map_err(|_| {
1574                                FttsError::Generic(format!(
1575                                    "generated code {code} does not fit the codec's i32"
1576                                ))
1577                            })?);
1578                        }
1579                        packet_frames += 1;
1580                        if packet_frames == PACKET_FRAMES {
1581                            codec
1582                                .stream_push(&mut state, &packet, packet_frames, &mut packet_pcm)
1583                                .map_err(checkpoint_error)?;
1584                            pcm.extend_from_slice(&packet_pcm);
1585                            first_audio_at.get_or_insert_with(|| synthesis_started.elapsed());
1586                            packet.clear();
1587                            packet_frames = 0;
1588                        }
1589                    }
1590                    if packet_frames > 0 {
1591                        codec
1592                            .stream_push(&mut state, &packet, packet_frames, &mut packet_pcm)
1593                            .map_err(checkpoint_error)?;
1594                        pcm.extend_from_slice(&packet_pcm);
1595                        first_audio_at.get_or_insert_with(|| synthesis_started.elapsed());
1596                    }
1597                    Ok((pcm, first_audio_at))
1598                },
1599            );
1600
1601            let mut tee = TeeGenerator {
1602                inner: &mut generator,
1603                frames: frame_tx,
1604            };
1605            let result = engine
1606                .synthesize(
1607                    request.clone(),
1608                    &preparer,
1609                    &mut tee as &mut dyn FrameGenerator,
1610                    cancellation,
1611                    observer,
1612                )
1613                .map_err(engine_error);
1614            drop(tee); // closes the channel; the worker drains the tail packet and exits
1615            let pcm = worker.join().expect("codec worker must not panic");
1616            // The engine's error wins the report: when generation fails, the worker usually
1617            // fails too (starved or fed a partial stream), and its complaint would bury the
1618            // actual cause.
1619            let result = result?;
1620            let (pcm, ttfa) = pcm?;
1621            Ok((result, pcm, ttfa))
1622        },
1623    )?;
1624
1625    if result.code_frames.is_empty() {
1626        return Err(FttsError::Generic(
1627            "the talker stopped before emitting a frame; there is no audio to write. This is a \
1628             model or prompt problem, not an output problem — check the speaker vector and the \
1629             text"
1630                .to_owned(),
1631        ));
1632    }
1633
1634    Ok(SynthesizedAudio {
1635        frames: result.generated_frames,
1636        prepared_token_count: result.prepared_token_count,
1637        pcm,
1638        ttfa,
1639    })
1640}
1641
1642/// Forwards a generator's frames unchanged while teeing each one to the codec worker.
1643///
1644/// A send only fails when the worker has already died with its own error; surfacing a
1645/// generation error here aborts the engine loop early, and the worker's real failure is
1646/// reported at join.
1647struct TeeGenerator<'a> {
1648    inner: &'a mut dyn FrameGenerator,
1649    frames: std::sync::mpsc::SyncSender<ftts_core::CodeFrame>,
1650}
1651
1652impl FrameGenerator for TeeGenerator<'_> {
1653    fn begin_utterance(&mut self, prepared: &PreparedText) -> Result<(), GenerationError> {
1654        self.inner.begin_utterance(prepared)
1655    }
1656
1657    fn next_frame(&mut self) -> Result<Option<ftts_core::CodeFrame>, GenerationError> {
1658        let frame = self.inner.next_frame()?;
1659        if let Some(frame) = &frame
1660            && self.frames.send(frame.clone()).is_err()
1661        {
1662            return Err(GenerationError::new(
1663                "the codec worker stopped accepting frames; its error follows at join",
1664            ));
1665        }
1666        Ok(frame)
1667    }
1668}
1669
1670/// Map an engine refusal onto the CLI's exit-code contract.
1671fn engine_error(error: EngineError) -> FttsError {
1672    match error {
1673        EngineError::BudgetExceeded(_) => FttsError::BudgetTimeout(error.to_string()),
1674        EngineError::ResourceAdmission(_) => FttsError::BudgetTimeout(error.to_string()),
1675        EngineError::TextPreparation(_) => FttsError::Input(error.to_string()),
1676        other => FttsError::Generic(other.to_string()),
1677    }
1678}
1679
1680/// A model-side failure, for callers that need the engine's own error type.
1681#[must_use]
1682pub fn generation_error(message: &str) -> GenerationError {
1683    GenerationError::new(message)
1684}
1685
1686#[cfg(test)]
1687mod tests {
1688    use super::*;
1689
1690    /// Audio already at the pinned rate must come back untouched — the resample path is additive
1691    /// and may not perturb any enrollment that worked before it existed.
1692    #[test]
1693    fn audio_at_the_pinned_rate_is_returned_bit_for_bit() {
1694        let pcm: Vec<f32> = (0..4_096)
1695            .map(|n| (n as f32 * 0.017).sin() * 0.4 + (n as f32 * 0.31).sin() * 0.05)
1696            .collect();
1697        let out = resample_to_speaker_rate(pcm.clone(), SPEAKER_SAMPLE_RATE_HZ);
1698        assert_eq!(out.len(), pcm.len());
1699        for (index, (a, b)) in out.iter().zip(pcm.iter()).enumerate() {
1700            assert!(
1701                a.to_bits() == b.to_bits(),
1702                "sample {index} was altered at the pinned rate"
1703            );
1704        }
1705    }
1706
1707    /// `E₁` sets the MMSE-LSA gain at every bin of every frame, so a mistyped coefficient would
1708    /// quietly bias the whole denoiser instead of failing. References computed from the
1709    /// convergent series `−γ − ln x + Σ (−1)^{k+1} x^k /(k·k!)`, a different algorithm from the
1710    /// rational fits under test.
1711    ///
1712    /// The tolerance is tight on purpose: a single-digit slip in the fifth-order coefficient
1713    /// perturbs `E₁(0.9)` by ~7.6e-7, which a looser bound would wave through.
1714    #[test]
1715    fn the_exponential_integral_matches_its_series_expansion() {
1716        // (x, E₁(x)) — the series branch, x < 1.
1717        for (x, expected) in [
1718            (0.1_f32, 1.822_923_9_f32),
1719            (0.5, 0.559_773_6),
1720            (0.9, 0.260_183_94),
1721        ] {
1722            let actual = exponential_integral_e1(x);
1723            let relative = ((actual - expected) / expected).abs();
1724            assert!(
1725                relative < 1e-6,
1726                "E1({x}) = {actual} but the series gives {expected} (relative {relative:e})"
1727            );
1728        }
1729
1730        // E₁ is positive and strictly decreasing; the two branches must agree where they meet.
1731        let below = exponential_integral_e1(0.999_9);
1732        let above = exponential_integral_e1(1.000_1);
1733        assert!(
1734            below > above && (below - above).abs() < 1e-4,
1735            "the series and rational branches disagree across x = 1: {below} vs {above}"
1736        );
1737        assert_eq!(
1738            exponential_integral_e1(0.0),
1739            0.0,
1740            "a non-positive argument must not produce NaN"
1741        );
1742    }
1743
1744    /// The denoiser has to do both halves of its job: drop the noise floor between bursts, and
1745    /// leave the signal itself standing. A filter that achieves the first by attenuating
1746    /// everything would pass a floor-only check while destroying the voice it was meant to clean.
1747    #[test]
1748    fn denoise_lowers_the_floor_between_bursts_without_eating_the_signal() {
1749        const TONE_HZ: f64 = 700.0;
1750        let samples = SPEAKER_SAMPLE_RATE_HZ as usize * 2;
1751        // Deterministic hiss, so the assertion cannot flake on a lucky seed.
1752        let mut state = 0x2545_F491_4F6C_DD1D_u64;
1753        let mut noise = || {
1754            state ^= state << 13;
1755            state ^= state >> 7;
1756            state ^= state << 17;
1757            ((state >> 40) as f32 / 16_777_216.0) - 0.5
1758        };
1759
1760        // Half-second bursts of tone alternating with silence, all of it under hiss.
1761        let clean: Vec<f32> = (0..samples)
1762            .map(|n| {
1763                let t = n as f64 / f64::from(SPEAKER_SAMPLE_RATE_HZ);
1764                let speaking = (n / (SPEAKER_SAMPLE_RATE_HZ as usize / 2)).is_multiple_of(2);
1765                if speaking {
1766                    (std::f64::consts::TAU * TONE_HZ * t).sin() as f32 * 0.35
1767                } else {
1768                    0.0
1769                }
1770            })
1771            .collect();
1772        // Hiss at ~-26 dBFS against a 0.35 tone (~17 dB SNR): the audible-voice-memo regime
1773        // this lever exists for. (An earlier revision's generator bug made the "hiss" 4000x
1774        // louder than the signal, and the assertions below were calibrated against artifacts.)
1775        let noisy: Vec<f32> = clean.iter().map(|s| s + noise() * 0.1).collect();
1776
1777        let cleaned = denoise_reference(&noisy);
1778        assert_eq!(cleaned.len(), noisy.len(), "denoise must preserve length");
1779        assert!(
1780            cleaned.iter().all(|s| s.is_finite()),
1781            "denoise produced a non-finite sample"
1782        );
1783
1784        let before = pause_floor_dbfs(&noisy);
1785        let after = pause_floor_dbfs(&cleaned);
1786        assert!(
1787            after < before - 3.0,
1788            "expected the pause floor to drop by >3 dB, got {before:.1} -> {after:.1} dBFS"
1789        );
1790
1791        // Energy inside a burst must survive. Compare the loudest quarter-second of each.
1792        let span = SPEAKER_SAMPLE_RATE_HZ as usize / 4;
1793        let peak_rms = |pcm: &[f32]| {
1794            pcm.chunks_exact(span)
1795                .map(|c| (c.iter().map(|s| s * s).sum::<f32>() / c.len() as f32).sqrt())
1796                .fold(0.0_f32, f32::max)
1797        };
1798        let kept = peak_rms(&cleaned) / peak_rms(&noisy);
1799        assert!(
1800            kept > 0.7,
1801            "denoise removed too much of the signal: peak RMS kept {kept:.3} of the original"
1802        );
1803    }
1804
1805    /// A reference that opens on speech, with no leading room tone to learn from, must keep that
1806    /// opening — most voice memos start the moment recording does.
1807    ///
1808    /// The probe is deliberately voice-*like*: a harmonic stack with vibrato and an amplitude
1809    /// envelope. That matters, because a steady unvarying partial is spectrally what a hum is,
1810    /// and suppressing it is correct behaviour rather than a bug — an earlier version of this
1811    /// test used a bare sine and was measuring the denoiser doing its job.
1812    #[test]
1813    fn denoise_keeps_a_reference_that_opens_on_speech() {
1814        let rate = SPEAKER_SAMPLE_RATE_HZ as usize;
1815        let mut state = 0x9E37_79B9_7F4A_7C15_u64;
1816        let mut hiss = || {
1817            state ^= state << 13;
1818            state ^= state >> 7;
1819            state ^= state << 17;
1820            ((state >> 40) as f32 / 16_777_216.0) - 0.5
1821        };
1822        let burst = rate / 4;
1823        let noisy: Vec<f32> = (0..rate * 3)
1824            .map(|n| {
1825                let t = n as f64 / rate as f64;
1826                let index = n / burst;
1827                let voice = if index.is_multiple_of(2) {
1828                    // Vibrato and a syllable envelope keep the partials moving, which is what
1829                    // distinguishes a voice from a tone to any minimum/quantile noise estimator.
1830                    let vibrato = 1.0 + 0.03 * (std::f64::consts::TAU * 5.5 * t).sin();
1831                    let phase = (n % burst) as f32 / burst as f32;
1832                    let envelope = (std::f32::consts::PI * phase).sin();
1833                    let f0 = 140.0 * vibrato * (1.0 + 0.15 * (index / 2) as f64);
1834                    (1..=10)
1835                        .map(|h| {
1836                            let a = 0.3 / h as f32;
1837                            (std::f64::consts::TAU * f0 * h as f64 * t).sin() as f32 * a
1838                        })
1839                        .sum::<f32>()
1840                        * envelope
1841                } else {
1842                    0.0
1843                };
1844                voice + hiss() * 0.02
1845            })
1846            .collect();
1847
1848        let cleaned = denoise_reference(&noisy);
1849        let rms = |pcm: &[f32]| (pcm.iter().map(|s| s * s).sum::<f32>() / pcm.len() as f32).sqrt();
1850        let kept = rms(&cleaned[..burst]) / rms(&noisy[..burst]);
1851        assert!(
1852            kept > 0.7,
1853            "the opening burst kept only {kept:.3} of its energy; the noise floor is being \
1854             seeded from speech the estimator has not yet learned to exclude"
1855        );
1856    }
1857
1858    /// Denoising must not buy a quiet floor by dulling the voice.
1859    ///
1860    /// High frequencies are where this fails first and where it matters most: broadband hiss
1861    /// overlaps sibilance almost exactly, so a suppressor tuned by overall SNR happily trades
1862    /// away 4–10 kHz — and the speaker encoder reads sibilance as identity, so that trade shows
1863    /// up as a duller *and less recognizable* clone rather than merely a duller one.
1864    ///
1865    /// The probe is broadband speech-like content: every band must survive comparably, so the
1866    /// assertion is on the spread across bands, not on any single band's absolute retention.
1867    #[test]
1868    fn denoise_does_not_preferentially_zap_high_frequencies() {
1869        let rate = SPEAKER_SAMPLE_RATE_HZ as usize;
1870        let mut state = 0xDEAD_BEEF_1234_5678_u64;
1871        let mut hiss = || {
1872            state ^= state << 13;
1873            state ^= state >> 7;
1874            state ^= state << 17;
1875            ((state >> 40) as f32 / 16_777_216.0) - 0.5
1876        };
1877        // Equal-amplitude tones spanning the band, gated into syllable-like bursts so the
1878        // estimator treats them as speech rather than as hum.
1879        let probes: [f64; 5] = [300.0, 1_200.0, 3_000.0, 6_000.0, 9_000.0];
1880        let burst = rate / 4;
1881        let noisy: Vec<f32> = (0..rate * 3)
1882            .map(|n| {
1883                let t = n as f64 / rate as f64;
1884                let voice = if (n / burst).is_multiple_of(2) {
1885                    let phase = (n % burst) as f32 / burst as f32;
1886                    let envelope = (std::f32::consts::PI * phase).sin();
1887                    probes
1888                        .iter()
1889                        .map(|hz| (std::f64::consts::TAU * hz * t).sin() as f32 * 0.12)
1890                        .sum::<f32>()
1891                        * envelope
1892                } else {
1893                    0.0
1894                };
1895                voice + hiss() * 0.02
1896            })
1897            .collect();
1898
1899        let cleaned = denoise_reference(&noisy);
1900
1901        // Per-probe retention, measured by projecting each band onto its own tone (a one-bin
1902        // Goertzel-style correlation) inside a burst.
1903        let span = burst / 2..burst;
1904        let energy_at = |pcm: &[f32], hz: f64| -> f32 {
1905            let (mut re, mut im) = (0.0_f64, 0.0_f64);
1906            for (offset, sample) in pcm[span.clone()].iter().enumerate() {
1907                let t = (span.start + offset) as f64 / rate as f64;
1908                let angle = std::f64::consts::TAU * hz * t;
1909                re += f64::from(*sample) * angle.cos();
1910                im += f64::from(*sample) * angle.sin();
1911            }
1912            (re.hypot(im) / span.len() as f64) as f32
1913        };
1914
1915        let retention: Vec<f32> = probes
1916            .iter()
1917            .map(|hz| energy_at(&cleaned, *hz) / energy_at(&noisy, *hz).max(1e-9))
1918            .collect();
1919        let low = retention[0];
1920        for (hz, kept) in probes.iter().zip(retention.iter()) {
1921            assert!(
1922                *kept > 0.5,
1923                "{hz} Hz retained only {kept:.3}; the denoiser is eating the band, not the noise \
1924                 (all bands: {retention:?})"
1925            );
1926            assert!(
1927                *kept > low * 0.6,
1928                "{hz} Hz retained {kept:.3} against {low:.3} at 300 Hz — high frequencies are \
1929                 being attenuated preferentially, which is how sibilance and speaker identity go \
1930                 (all bands: {retention:?})"
1931            );
1932        }
1933    }
1934
1935    /// A clip too short to survive its own downsample must round to nothing here rather than
1936    /// reaching the mel front end as an empty slice — which is why `decode_reference_audio`
1937    /// re-checks emptiness against the resampled PCM instead of only the decoded PCM.
1938    #[test]
1939    fn a_clip_shorter_than_its_downsample_ratio_resamples_to_nothing() {
1940        let out = resample_to_speaker_rate(vec![0.25], 192_000);
1941        assert!(
1942            out.is_empty(),
1943            "one sample at 192 kHz is less than half an output sample at \
1944             {SPEAKER_SAMPLE_RATE_HZ} Hz, so it cannot produce one"
1945        );
1946    }
1947
1948    /// A tone that survives the resample proves the kernel is a real lowpass and not a decimator:
1949    /// 48 kHz is the rate every phone and Mac voice memo records at, and a 1 kHz tone sits well
1950    /// inside the 12 kHz band that survives the trip to 24 kHz.
1951    #[test]
1952    fn a_48k_tone_resamples_to_24k_with_its_shape_intact() {
1953        const SOURCE_HZ: u32 = 48_000;
1954        const TONE_HZ: f64 = 1_000.0;
1955        let samples = SOURCE_HZ as usize; // one second
1956        let pcm: Vec<f32> = (0..samples)
1957            .map(|n| {
1958                let t = n as f64 / f64::from(SOURCE_HZ);
1959                (std::f64::consts::TAU * TONE_HZ * t).sin() as f32
1960            })
1961            .collect();
1962
1963        let out = resample_to_speaker_rate(pcm, SOURCE_HZ);
1964
1965        let expected_len = SPEAKER_SAMPLE_RATE_HZ as usize;
1966        assert!(
1967            out.len().abs_diff(expected_len) <= 1,
1968            "expected ~{expected_len} samples at {SPEAKER_SAMPLE_RATE_HZ} Hz, got {}",
1969            out.len()
1970        );
1971
1972        // Compare against the tone sampled directly at the target rate, ignoring the window's
1973        // run-up at each end where the kernel is truncated by the signal boundary.
1974        let skip = 64;
1975        let interior = out.len() - skip;
1976        let mut worst = 0.0_f32;
1977        for (index, sample) in out.iter().enumerate().take(interior).skip(skip) {
1978            let t = index as f64 / f64::from(SPEAKER_SAMPLE_RATE_HZ);
1979            let ideal = (std::f64::consts::TAU * TONE_HZ * t).sin() as f32;
1980            worst = worst.max((sample - ideal).abs());
1981        }
1982        assert!(
1983            worst < 0.02,
1984            "resampled tone drifted from the analytic reference by {worst}"
1985        );
1986    }
1987
1988    #[test]
1989    fn a_short_speaker_vector_is_refused_rather_than_padded() {
1990        let dir = std::env::temp_dir().join("ftts-synth-tests");
1991        fs::create_dir_all(&dir).expect("temp dir");
1992        let path = dir.join("short.spk");
1993        fs::write(&path, vec![0u8; 64]).expect("write");
1994        let error = read_speaker_vector(&path).expect_err("a short vector must be refused");
1995        let message = error.to_string();
1996        assert!(message.contains("64 bytes"), "{message}");
1997        assert!(message.contains("4096"), "{message}");
1998    }
1999
2000    #[test]
2001    fn a_non_finite_speaker_vector_is_refused() {
2002        let dir = std::env::temp_dir().join("ftts-synth-tests");
2003        fs::create_dir_all(&dir).expect("temp dir");
2004        let path = dir.join("nan.spk");
2005        let mut bytes = vec![0u8; SPEAKER_VECTOR_BYTES];
2006        bytes[0..4].copy_from_slice(&f32::NAN.to_le_bytes());
2007        fs::write(&path, &bytes).expect("write");
2008        let error = read_speaker_vector(&path).expect_err("NaN must be refused");
2009        assert!(error.to_string().contains("index 0"), "{error}");
2010    }
2011
2012    #[test]
2013    fn a_well_formed_speaker_vector_reads_back_exactly() {
2014        let dir = std::env::temp_dir().join("ftts-synth-tests");
2015        fs::create_dir_all(&dir).expect("temp dir");
2016        let path = dir.join("good.spk");
2017        let expected: Vec<f32> = (0..TALKER_HIDDEN).map(|i| i as f32 * 0.001).collect();
2018        let mut bytes = Vec::with_capacity(SPEAKER_VECTOR_BYTES);
2019        for value in &expected {
2020            bytes.extend_from_slice(&value.to_le_bytes());
2021        }
2022        fs::write(&path, &bytes).expect("write");
2023        assert_eq!(read_speaker_vector(&path).expect("read"), expected);
2024    }
2025
2026    #[test]
2027    fn enrollment_writer_refuses_overwrite_and_preserves_the_vector() {
2028        let path = std::env::temp_dir().join(format!(
2029            "ftts-enroll-{}-{}.spk",
2030            std::process::id(),
2031            std::time::SystemTime::now()
2032                .duration_since(std::time::UNIX_EPOCH)
2033                .expect("clock")
2034                .as_nanos()
2035        ));
2036        let expected: Vec<f32> = (0..TALKER_HIDDEN)
2037            .map(|index| index as f32 * 0.125)
2038            .collect();
2039        write_speaker_vector_new(&path, &expected).expect("initial enrollment write");
2040        assert_eq!(
2041            read_speaker_vector(&path).expect("read enrolled vector"),
2042            expected
2043        );
2044        let error = write_speaker_vector_new(&path, &[0.0; TALKER_HIDDEN])
2045            .expect_err("an enrollment must never replace an existing voice");
2046        assert!(error.to_string().contains("without overwriting"), "{error}");
2047    }
2048
2049    #[test]
2050    fn wav_reference_decodes_to_mono_24khz_pcm() {
2051        let path = std::env::temp_dir().join(format!(
2052            "ftts-reference-{}-{}.wav",
2053            std::process::id(),
2054            std::time::SystemTime::now()
2055                .duration_since(std::time::UNIX_EPOCH)
2056                .expect("clock")
2057                .as_nanos()
2058        ));
2059        let pcm: Vec<f32> = (0..1_920)
2060            .map(|index| (index as f32 / 1_920.0 * std::f32::consts::TAU).sin() * 0.25)
2061            .collect();
2062        fs::write(
2063            &path,
2064            ftts_core::audio::encode_wav(&pcm, SPEAKER_SAMPLE_RATE_HZ),
2065        )
2066        .expect("write reference WAV");
2067        let decoded = decode_reference_audio(&path).expect("decode reference WAV");
2068        assert_eq!(decoded.len(), pcm.len());
2069        assert!(decoded.iter().all(|sample| sample.is_finite()));
2070    }
2071
2072    #[test]
2073    fn a_bundle_names_the_file_that_is_actually_missing() {
2074        // An agent that gets "model not found" for a directory holding three of four files cannot
2075        // act on it; the message must name the one that is absent.
2076        let dir = std::env::temp_dir().join("ftts-bundle-tests-empty");
2077        fs::create_dir_all(&dir).expect("temp dir");
2078        let error = ModelBundle::resolve(&dir).expect_err("an empty directory is not a bundle");
2079        assert!(error.to_string().contains("model.safetensors"), "{error}");
2080    }
2081
2082    #[test]
2083    fn a_complete_bundle_prefers_its_canonical_artifact_for_synthesis() {
2084        let nonce = std::time::SystemTime::now()
2085            .duration_since(std::time::UNIX_EPOCH)
2086            .expect("clock after epoch")
2087            .as_nanos();
2088        let dir = std::env::temp_dir().join(format!(
2089            "ftts-bundle-canonical-{}-{nonce}",
2090            std::process::id()
2091        ));
2092        fs::create_dir_all(dir.join("speech_tokenizer")).expect("create bundle sidecar directory");
2093        for name in [
2094            CANONICAL_MODEL_BASENAME,
2095            "speech_tokenizer/model.safetensors",
2096            "vocab.json",
2097            "merges.txt",
2098            "tokenizer_config.json",
2099        ] {
2100            fs::write(dir.join(name), []).expect("write bundle fixture sidecar");
2101        }
2102
2103        let expected_artifact = dir.join(CANONICAL_MODEL_BASENAME);
2104        let bundle = ModelBundle::resolve(&dir).expect("complete canonical bundle resolves");
2105        assert_eq!(
2106            bundle.canonical_main.as_deref(),
2107            Some(expected_artifact.as_path())
2108        );
2109        assert!(
2110            !bundle.main.exists(),
2111            "canonical synthesis must not require the raw main checkpoint"
2112        );
2113
2114        let explicit = ModelBundle::resolve(&expected_artifact)
2115            .expect("an explicit canonical artifact resolves against its sidecars");
2116        assert_eq!(
2117            explicit.canonical_main.as_deref(),
2118            Some(expected_artifact.as_path())
2119        );
2120    }
2121}