use crate::error::FttsError;
use ftts_core::{
CancellationToken, EngineError, FrameGenerator, GenerationError, NormalizationOptions,
NormalizationTrace, PreparedText, SynthesisObserver, SynthesisRequest, TextPreparationError,
TextPreparer, TtsEngine,
};
use ftts_model_qwen::checkpoint::{
CODEC_LANGUAGE_ENGLISH_ID, CheckpointError, CodecCheckpoint, TALKER_HIDDEN, TalkerCheckpoint,
};
use ftts_model_qwen::generate::{QwenGenerator, QwenGeneratorConfig};
use ftts_model_qwen::microdecoder::MicrodecoderConfig;
use ftts_model_qwen::prompt::{CloneMode, PromptMode};
use ftts_model_qwen::sampler::SamplingMode;
use ftts_model_qwen::speaker::{
Encoder as SpeakerEncoder, SPEAKER_SAMPLE_RATE_HZ, log_mel_from_24khz_pcm,
};
use ftts_model_qwen::talker::TalkerConfig;
use ftts_model_qwen::tokenizer::{QwenTokenizer, TokenizerFiles};
use std::fs;
use std::fs::OpenOptions;
use std::path::{Path, PathBuf};
use symphonia::core::audio::SampleBuffer;
use symphonia::core::codecs::DecoderOptions;
use symphonia::core::errors::Error as SymphoniaError;
use symphonia::core::formats::FormatOptions;
use symphonia::core::io::MediaSourceStream;
use symphonia::core::meta::MetadataOptions;
use symphonia::core::probe::Hint;
use symphonia::default::{get_codecs, get_probe};
pub const SPEAKER_VECTOR_BYTES: usize = TALKER_HIDDEN * 4;
const CANONICAL_MODEL_BASENAME: &str = "qwen3-tts-12hz-0.6b-base.fttsq";
fn checkpoint_error(error: CheckpointError) -> FttsError {
FttsError::ArtifactFormat(error.to_string())
}
#[derive(Clone, Debug)]
pub struct ModelBundle {
pub root: PathBuf,
pub main: PathBuf,
pub canonical_main: Option<PathBuf>,
pub codec: PathBuf,
}
impl ModelBundle {
pub fn resolve(model: &Path) -> Result<Self, FttsError> {
let root = if model.is_dir() {
model.to_path_buf()
} else {
model
.parent()
.ok_or_else(|| {
FttsError::ModelNotFound(format!(
"model path {} has no parent directory",
model.display()
))
})?
.to_path_buf()
};
let canonical_main = if model.is_dir() {
let canonical = root.join(CANONICAL_MODEL_BASENAME);
if canonical.is_file() {
Some(canonical)
} else {
None
}
} else if model.extension().and_then(|extension| extension.to_str()) == Some("fttsq") {
Some(model.to_path_buf())
} else {
None
};
let main = root.join("model.safetensors");
let codec = root.join("speech_tokenizer/model.safetensors");
let (main_label, main_path) = match canonical_main.as_ref() {
Some(path) => ("canonical talker artifact", path),
None => ("talker checkpoint", &main),
};
for (label, path) in [
(main_label, main_path),
("codec checkpoint", &codec),
("tokenizer vocabulary", &root.join("vocab.json")),
("tokenizer merges", &root.join("merges.txt")),
("tokenizer config", &root.join("tokenizer_config.json")),
] {
if !path.is_file() {
return Err(FttsError::ModelNotFound(format!(
"{label} is missing at {}; `ftts say` needs a complete model directory \
({CANONICAL_MODEL_BASENAME} or model.safetensors, \
speech_tokenizer/model.safetensors, \
vocab.json, merges.txt, tokenizer_config.json)",
path.display()
)));
}
}
Ok(Self {
root,
main,
canonical_main,
codec,
})
}
}
pub struct LoadedModel {
talker: TalkerCheckpoint,
codec: CodecCheckpoint,
tokenizer: QwenTokenizer,
}
impl LoadedModel {
pub fn load(bundle: &ModelBundle) -> Result<Self, FttsError> {
let read = |name: &str| -> Result<String, FttsError> {
let path = bundle.root.join(name);
fs::read_to_string(&path).map_err(|error| {
FttsError::ArtifactFormat(format!("cannot read {}: {error}", path.display()))
})
};
let vocab = read("vocab.json")?;
let merges = read("merges.txt")?;
let config = read("tokenizer_config.json")?;
let tokenizer = QwenTokenizer::from_files_using_environment(TokenizerFiles {
vocab_json: &vocab,
merges_txt: &merges,
tokenizer_config_json: &config,
})
.map_err(|error| FttsError::ArtifactFormat(format!("tokenizer unusable: {error}")))?;
Ok(Self {
talker: match bundle.canonical_main.as_deref() {
Some(path) => TalkerCheckpoint::load_fttsq(path).map_err(checkpoint_error)?,
None => TalkerCheckpoint::load(&bundle.main).map_err(checkpoint_error)?,
},
codec: CodecCheckpoint::load(&bundle.codec).map_err(checkpoint_error)?,
tokenizer,
})
}
}
pub fn read_speaker_vector(path: &Path) -> Result<Vec<f32>, FttsError> {
let bytes = fs::read(path).map_err(|error| {
FttsError::Input(format!(
"cannot read speaker vector {}: {error}",
path.display()
))
})?;
if bytes.len() != SPEAKER_VECTOR_BYTES {
return Err(FttsError::Input(format!(
"speaker vector {} is {} bytes; `ftts say --voice` expects exactly {} \
({TALKER_HIDDEN} little-endian f32)",
path.display(),
bytes.len(),
SPEAKER_VECTOR_BYTES
)));
}
let vector: Vec<f32> = bytes
.as_chunks::<4>()
.0
.iter()
.map(|quad| f32::from_le_bytes(*quad))
.collect();
if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
return Err(FttsError::Input(format!(
"speaker vector {} holds a non-finite value at index {index}; it would poison every \
prefill position it is summed into",
path.display()
)));
}
Ok(vector)
}
pub fn speaker_from_voice(bundle: &ModelBundle, path: &Path) -> Result<Vec<f32>, FttsError> {
let bytes = fs::read(path).map_err(|error| {
FttsError::Input(format!(
"cannot read voice source {}: {error}",
path.display()
))
})?;
if bytes.len() == SPEAKER_VECTOR_BYTES {
return decode_speaker_vector(path, &bytes);
}
let pcm = decode_reference_audio_any(path)?;
let mel = log_mel_from_24khz_pcm(&pcm)
.map_err(|error| FttsError::Input(format!("cannot extract speaker features: {error}")))?;
let encoder = match bundle.canonical_main.as_deref() {
Some(artifact) => SpeakerEncoder::load_fttsq(artifact),
None => SpeakerEncoder::load(&bundle.main),
}
.map_err(checkpoint_error)?;
let vector = encoder.encode(&mel.values, mel.frames);
if vector.iter().all(|value| value.is_finite()) {
Ok(vector)
} else {
Err(FttsError::Input(
"speaker encoder produced a non-finite x-vector; refusing to condition synthesis"
.to_owned(),
))
}
}
pub fn write_speaker_vector_new(path: &Path, vector: &[f32]) -> Result<(), FttsError> {
if vector.len() != TALKER_HIDDEN {
return Err(FttsError::Input(format!(
"cannot write {}-wide speaker vector; expected {TALKER_HIDDEN}",
vector.len()
)));
}
if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
return Err(FttsError::Input(format!(
"cannot write speaker vector with a non-finite value at index {index}"
)));
}
let mut bytes = Vec::with_capacity(SPEAKER_VECTOR_BYTES);
for value in vector {
bytes.extend_from_slice(&value.to_le_bytes());
}
use std::io::Write;
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
.map_err(|error| {
FttsError::Input(format!(
"cannot create enrolled voice {} without overwriting an existing file: {error}",
path.display()
))
})?;
file.write_all(&bytes).map_err(|error| {
FttsError::Input(format!(
"cannot write enrolled voice {}: {error}",
path.display()
))
})
}
fn decode_speaker_vector(path: &Path, bytes: &[u8]) -> Result<Vec<f32>, FttsError> {
let vector: Vec<f32> = bytes
.as_chunks::<4>()
.0
.iter()
.map(|quad| f32::from_le_bytes(*quad))
.collect();
if let Some(index) = vector.iter().position(|value| !value.is_finite()) {
return Err(FttsError::Input(format!(
"speaker vector {} holds a non-finite value at index {index}; it would poison every \
prefill position it is summed into",
path.display()
)));
}
Ok(vector)
}
const SYSTEM_DECODED_EXTENSIONS: [&str; 6] = ["m4a", "mp3", "aac", "mp4", "ogg", "opus"];
fn decode_reference_audio_any(path: &Path) -> Result<Vec<f32>, FttsError> {
let extension = path
.extension()
.and_then(|extension| extension.to_str())
.map(str::to_ascii_lowercase);
let needs_system_decoder = extension
.as_deref()
.is_some_and(|extension| SYSTEM_DECODED_EXTENSIONS.contains(&extension));
if !needs_system_decoder {
return decode_reference_audio(path);
}
let staging = std::env::temp_dir().join(format!(
"ftts-enroll-{}-{}.wav",
std::process::id(),
path.file_stem()
.and_then(|stem| stem.to_str())
.unwrap_or("reference")
));
let attempts: &[(&str, Vec<&std::ffi::OsStr>)] = &[
(
"afconvert",
vec![
"-f".as_ref(),
"WAVE".as_ref(),
"-d".as_ref(),
"LEI16@24000".as_ref(),
"-c".as_ref(),
"1".as_ref(),
path.as_os_str(),
staging.as_os_str(),
],
),
(
"ffmpeg",
vec![
"-y".as_ref(),
"-loglevel".as_ref(),
"error".as_ref(),
"-i".as_ref(),
path.as_os_str(),
"-acodec".as_ref(),
"pcm_s16le".as_ref(),
"-ar".as_ref(),
"24000".as_ref(),
"-ac".as_ref(),
"1".as_ref(),
staging.as_os_str(),
],
),
];
let mut ran = false;
for (tool, arguments) in attempts {
match std::process::Command::new(tool).args(arguments).status() {
Ok(status) if status.success() => {
ran = true;
break;
}
Ok(status) => {
let _ = fs::remove_file(&staging);
return Err(FttsError::Input(format!(
"{tool} failed decoding reference audio {} (exit {status})",
path.display()
)));
}
Err(_) => continue, }
}
if !ran {
return Err(FttsError::Input(format!(
"reference audio {} is a compressed container and no system decoder was found; \
install afconvert (macOS) or ffmpeg, or supply WAV/FLAC",
path.display()
)));
}
let decoded = decode_reference_audio(&staging);
let _ = fs::remove_file(&staging);
decoded
}
fn decode_reference_audio(path: &Path) -> Result<Vec<f32>, FttsError> {
let file = fs::File::open(path).map_err(|error| {
FttsError::Input(format!(
"cannot open reference audio {}: {error}",
path.display()
))
})?;
let mut hint = Hint::new();
if let Some(extension) = path.extension().and_then(|extension| extension.to_str()) {
hint.with_extension(extension);
}
let stream = MediaSourceStream::new(Box::new(file), Default::default());
let probed = get_probe()
.format(
&hint,
stream,
&FormatOptions::default(),
&MetadataOptions::default(),
)
.map_err(|error| {
FttsError::Input(format!(
"cannot identify reference audio {}: {error}",
path.display()
))
})?;
let mut format = probed.format;
let track = format.default_track().ok_or_else(|| {
FttsError::Input(format!(
"reference audio {} has no default audio track",
path.display()
))
})?;
let track_id = track.id;
let mut decoder = get_codecs()
.make(&track.codec_params, &DecoderOptions::default())
.map_err(|error| {
FttsError::Input(format!(
"cannot decode reference audio {}: {error}",
path.display()
))
})?;
let mut sample_rate = None;
let mut mono = Vec::new();
loop {
let packet = match format.next_packet() {
Ok(packet) => packet,
Err(SymphoniaError::IoError(error))
if error.kind() == std::io::ErrorKind::UnexpectedEof =>
{
break;
}
Err(error) => {
return Err(FttsError::Input(format!(
"cannot read reference audio {}: {error}",
path.display()
)));
}
};
if packet.track_id() != track_id {
continue;
}
let decoded = decoder.decode(&packet).map_err(|error| {
FttsError::Input(format!(
"cannot decode reference audio {}: {error}",
path.display()
))
})?;
let spec = *decoded.spec();
match sample_rate {
Some(rate) if rate != spec.rate => {
return Err(FttsError::Input(format!(
"reference audio {} changed sample rate mid-stream ({rate} to {} Hz)",
path.display(),
spec.rate
)));
}
None => sample_rate = Some(spec.rate),
Some(_) => {}
}
let channels = spec.channels.count();
let mut samples = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
samples.copy_interleaved_ref(decoded);
for frame in samples.samples().chunks_exact(channels) {
mono.push(frame.iter().sum::<f32>() / channels as f32);
}
}
let rate = sample_rate.ok_or_else(|| {
FttsError::Input(format!(
"reference audio {} contains no decodable samples",
path.display()
))
})?;
if rate != SPEAKER_SAMPLE_RATE_HZ {
return Err(FttsError::Input(format!(
"reference audio {} is {rate} Hz; the pinned speaker encoder requires {SPEAKER_SAMPLE_RATE_HZ} Hz",
path.display()
)));
}
if mono.is_empty() {
return Err(FttsError::Input(format!(
"reference audio {} contains no PCM samples",
path.display()
)));
}
Ok(mono)
}
struct PreparedPassThrough {
prepared: PreparedText,
}
impl TextPreparer for PreparedPassThrough {
fn prepare(
&self,
_text: &str,
_options: &NormalizationOptions,
) -> Result<PreparedText, TextPreparationError> {
Ok(PreparedText::new(
self.prepared.token_ids.clone(),
NormalizationTrace {
mode: self.prepared.normalization_trace.mode,
unicode_version: self.prepared.normalization_trace.unicode_version.clone(),
changes: self.prepared.normalization_trace.changes.clone(),
},
))
}
}
pub struct SynthesizedAudio {
pub frames: u64,
pub prepared_token_count: usize,
pub pcm: Vec<f32>,
}
#[allow(clippy::too_many_arguments)]
pub fn synthesize(
model: &LoadedModel,
engine: &TtsEngine,
request: &SynthesisRequest,
speaker: &[f32],
seed: u64,
cancellation: &CancellationToken,
observer: &dyn SynthesisObserver,
) -> Result<SynthesizedAudio, FttsError> {
let prepared_raw = model
.tokenizer
.prepare(&request.text, &request.normalization_options)
.map_err(|error| FttsError::Input(format!("text preparation failed: {error}")))?;
let wrapped = TalkerCheckpoint::wrap_target_ids(&prepared_raw.token_ids);
let prepared = PreparedText::new(wrapped.clone(), prepared_raw.normalization_trace);
let ids = TalkerCheckpoint::utterance_text_ids(&wrapped);
let table = model
.talker
.gather_text_rows(&ids)
.map_err(checkpoint_error)?;
let header = model
.talker
.xvector_header(&table, speaker, CODEC_LANGUAGE_ENGLISH_ID)
.map_err(checkpoint_error)?;
let tts_eos = model.talker.tts_eos(&table);
let talker_layers = model.talker.talker_layer_weights();
let micro_layers = model.talker.microdecoder_layer_weights();
let residual = model.talker.residual_embedding_slices();
let heads = model.talker.microdecoder_head_slices();
let micro_residual = &residual[..residual.len() - 1];
let mut generator = QwenGenerator::new(QwenGeneratorConfig {
talker_config: TalkerConfig::default(),
talker_weights: model.talker.talker_weights(&talker_layers),
text: model.talker.text_weights(&table),
feedback: model.talker.feedback_tables(&residual),
microdecoder_config: MicrodecoderConfig::default(),
microdecoder_weights: model.talker.microdecoder_weights(
µ_layers,
micro_residual,
&heads,
),
prompt_mode: PromptMode {
clone_mode: CloneMode::XVector,
non_streaming_mode: false,
},
header,
tts_eos,
reference: None,
sampling_mode: SamplingMode::Production,
seed,
});
let preparer = PreparedPassThrough { prepared };
let result = engine
.synthesize(
request.clone(),
&preparer,
&mut generator as &mut dyn FrameGenerator,
cancellation,
observer,
)
.map_err(engine_error)?;
if result.code_frames.is_empty() {
return Err(FttsError::Generic(
"the talker stopped before emitting a frame; there is no audio to write. This is a \
model or prompt problem, not an output problem — check the speaker vector and the \
text"
.to_owned(),
));
}
let frames = result.code_frames.len();
let mut codes = Vec::with_capacity(frames * 16);
for frame in &result.code_frames {
if frame.codes.len() != 16 {
return Err(FttsError::Generic(format!(
"generated frame carries {} codes, expected 16",
frame.codes.len()
)));
}
for code in &frame.codes {
codes.push(i32::try_from(*code).map_err(|_| {
FttsError::Generic(format!(
"generated code {code} does not fit the codec's i32"
))
})?);
}
}
let pcm = model
.codec
.decode(&codes, frames)
.map_err(checkpoint_error)?;
Ok(SynthesizedAudio {
frames: result.generated_frames,
prepared_token_count: result.prepared_token_count,
pcm,
})
}
fn engine_error(error: EngineError) -> FttsError {
match error {
EngineError::BudgetExceeded(_) => FttsError::BudgetTimeout(error.to_string()),
EngineError::ResourceAdmission(_) => FttsError::BudgetTimeout(error.to_string()),
EngineError::TextPreparation(_) => FttsError::Input(error.to_string()),
other => FttsError::Generic(other.to_string()),
}
}
#[must_use]
pub fn generation_error(message: &str) -> GenerationError {
GenerationError::new(message)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_short_speaker_vector_is_refused_rather_than_padded() {
let dir = std::env::temp_dir().join("ftts-synth-tests");
fs::create_dir_all(&dir).expect("temp dir");
let path = dir.join("short.spk");
fs::write(&path, vec![0u8; 64]).expect("write");
let error = read_speaker_vector(&path).expect_err("a short vector must be refused");
let message = error.to_string();
assert!(message.contains("64 bytes"), "{message}");
assert!(message.contains("4096"), "{message}");
}
#[test]
fn a_non_finite_speaker_vector_is_refused() {
let dir = std::env::temp_dir().join("ftts-synth-tests");
fs::create_dir_all(&dir).expect("temp dir");
let path = dir.join("nan.spk");
let mut bytes = vec![0u8; SPEAKER_VECTOR_BYTES];
bytes[0..4].copy_from_slice(&f32::NAN.to_le_bytes());
fs::write(&path, &bytes).expect("write");
let error = read_speaker_vector(&path).expect_err("NaN must be refused");
assert!(error.to_string().contains("index 0"), "{error}");
}
#[test]
fn a_well_formed_speaker_vector_reads_back_exactly() {
let dir = std::env::temp_dir().join("ftts-synth-tests");
fs::create_dir_all(&dir).expect("temp dir");
let path = dir.join("good.spk");
let expected: Vec<f32> = (0..TALKER_HIDDEN).map(|i| i as f32 * 0.001).collect();
let mut bytes = Vec::with_capacity(SPEAKER_VECTOR_BYTES);
for value in &expected {
bytes.extend_from_slice(&value.to_le_bytes());
}
fs::write(&path, &bytes).expect("write");
assert_eq!(read_speaker_vector(&path).expect("read"), expected);
}
#[test]
fn enrollment_writer_refuses_overwrite_and_preserves_the_vector() {
let path = std::env::temp_dir().join(format!(
"ftts-enroll-{}-{}.spk",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock")
.as_nanos()
));
let expected: Vec<f32> = (0..TALKER_HIDDEN)
.map(|index| index as f32 * 0.125)
.collect();
write_speaker_vector_new(&path, &expected).expect("initial enrollment write");
assert_eq!(
read_speaker_vector(&path).expect("read enrolled vector"),
expected
);
let error = write_speaker_vector_new(&path, &[0.0; TALKER_HIDDEN])
.expect_err("an enrollment must never replace an existing voice");
assert!(error.to_string().contains("without overwriting"), "{error}");
}
#[test]
fn wav_reference_decodes_to_mono_24khz_pcm() {
let path = std::env::temp_dir().join(format!(
"ftts-reference-{}-{}.wav",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock")
.as_nanos()
));
let pcm: Vec<f32> = (0..1_920)
.map(|index| (index as f32 / 1_920.0 * std::f32::consts::TAU).sin() * 0.25)
.collect();
fs::write(
&path,
ftts_core::audio::encode_wav(&pcm, SPEAKER_SAMPLE_RATE_HZ),
)
.expect("write reference WAV");
let decoded = decode_reference_audio(&path).expect("decode reference WAV");
assert_eq!(decoded.len(), pcm.len());
assert!(decoded.iter().all(|sample| sample.is_finite()));
}
#[test]
fn a_bundle_names_the_file_that_is_actually_missing() {
let dir = std::env::temp_dir().join("ftts-bundle-tests-empty");
fs::create_dir_all(&dir).expect("temp dir");
let error = ModelBundle::resolve(&dir).expect_err("an empty directory is not a bundle");
assert!(error.to_string().contains("model.safetensors"), "{error}");
}
#[test]
fn a_complete_bundle_prefers_its_canonical_artifact_for_synthesis() {
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock after epoch")
.as_nanos();
let dir = std::env::temp_dir().join(format!(
"ftts-bundle-canonical-{}-{nonce}",
std::process::id()
));
fs::create_dir_all(dir.join("speech_tokenizer")).expect("create bundle sidecar directory");
for name in [
CANONICAL_MODEL_BASENAME,
"speech_tokenizer/model.safetensors",
"vocab.json",
"merges.txt",
"tokenizer_config.json",
] {
fs::write(dir.join(name), []).expect("write bundle fixture sidecar");
}
let expected_artifact = dir.join(CANONICAL_MODEL_BASENAME);
let bundle = ModelBundle::resolve(&dir).expect("complete canonical bundle resolves");
assert_eq!(
bundle.canonical_main.as_deref(),
Some(expected_artifact.as_path())
);
assert!(
!bundle.main.exists(),
"canonical synthesis must not require the raw main checkpoint"
);
let explicit = ModelBundle::resolve(&expected_artifact)
.expect("an explicit canonical artifact resolves against its sidecars");
assert_eq!(
explicit.canonical_main.as_deref(),
Some(expected_artifact.as_path())
);
}
}