mod anchors;
mod search;
mod spec;
use std::ops::ControlFlow;
use loqui_audio::Pcm;
use self::search::{Evaluated, Evaluator, refine};
use self::spec::Lead;
use crate::speaker::{self, SpeakerEmbedding};
use crate::{Engine, Error};
pub const MAX_VOICES: usize = 6;
pub const MAX_EVALUATIONS: usize = 200;
pub const MIN_MATCH_SECS: f32 = 3.0;
const PROBE_CHARS: std::ops::RangeInclusive<usize> = 20..=200;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum Accent {
#[default]
Auto,
American,
British,
}
impl Accent {
fn family(self) -> Option<char> {
match self {
Self::Auto => None,
Self::American => Some('a'),
Self::British => Some('b'),
}
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct MatchRequest {
pub audio: Vec<u8>,
pub transcript: Option<String>,
pub max_voices: usize,
pub evaluations: usize,
pub accent: Accent,
}
impl MatchRequest {
pub fn new(audio: Vec<u8>) -> Self {
Self { audio, transcript: None, max_voices: 4, evaluations: 36, accent: Accent::Auto }
}
}
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct BlendMatch {
pub spec: String,
pub similarity: f32,
pub closest_voice: String,
pub closest_similarity: f32,
pub ranking: Vec<(String, f32)>,
pub speed: Option<f32>,
pub evaluations: usize,
}
#[derive(Debug, Clone, Copy, PartialEq)]
#[non_exhaustive]
pub enum MatchProgress {
Preparing { done: usize, total: usize },
Evaluated { evaluations: usize, budget: usize, best_similarity: f32 },
}
impl Engine {
pub fn match_voice(
&self,
request: &MatchRequest,
progress: &mut dyn FnMut(&MatchProgress) -> ControlFlow<()>,
) -> Result<BlendMatch, Error> {
if !(1..=MAX_VOICES).contains(&request.max_voices) {
return Err(Error::Invalid(format!("max_voices must be 1 to {MAX_VOICES}")));
}
if !(1..=MAX_EVALUATIONS).contains(&request.evaluations) {
return Err(Error::Invalid(format!("evaluations must be 1 to {MAX_EVALUATIONS}")));
}
let tts = self.inner.tts.as_ref().ok_or(Error::Disabled("text-to-speech"))?;
let speaker_slot = self.inner.speaker.as_ref().ok_or(Error::Disabled("speaker embeddings"))?;
let target_pcm = loqui_audio::decode_mono_16k(&request.audio, Some(self.inner.max_audio_secs))?;
let encoder = speaker_slot.get()?;
let target = speaker::embed(&encoder, &target_pcm.samples, MIN_MATCH_SECS)?;
let target_secs = speaker::voiced(&target_pcm.samples).0.len() as f32 / loqui_speaker::SAMPLE_RATE as f32;
self.fetch_voices()?;
let kokoro = tts.get()?;
let embed_spoken = |text: &str, spec: &str, speed: f32| -> Result<(SpeakerEmbedding, f32), Error> {
let blend = self.resolve(spec)?;
let samples = kokoro.speak(text, &blend, speed)?;
let secs = samples.len() as f32 / loqui_kokoro::SAMPLE_RATE as f32;
let pcm = loqui_audio::resample(Pcm { samples, rate: loqui_kokoro::SAMPLE_RATE }, loqui_speaker::SAMPLE_RATE)?;
Ok((speaker::embed(&encoder, &pcm.samples, 0.5)?, secs))
};
let voices = loqui_kokoro::ENGLISH_VOICES;
let anchors = self.anchors(&encoder, &mut |voice| embed_spoken(anchors::PROBE, voice, 1.0).map(|(e, _)| e), progress)?;
let mut ranking: Vec<(&str, f32)> =
voices.iter().zip(&anchors).map(|(voice, anchor)| (*voice, loqui_speaker::cosine(target.as_slice(), anchor))).collect();
ranking.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(b.0)));
let top: Vec<&str> = ranking.iter().take(request.max_voices).map(|(voice, _)| *voice).collect();
let nearest_of = |family: char| ranking.iter().map(|(voice, _)| *voice).find(|voice| voice.starts_with(family));
let transcript = request.transcript.as_deref().map(probe_text).filter(|t| PROBE_CHARS.contains(&t.chars().count()));
let (probe, speed) = match transcript {
Some(text) => {
let (_, secs) = embed_spoken(&text, top[0], 1.0)?;
let speed = (secs / target_secs.max(0.1)).clamp(0.75, 1.35);
(text, Some((speed * 20.0).round() / 20.0))
}
None => (anchors::PROBE.to_owned(), None),
};
let start = {
let index = |voice: &str| voices.iter().position(|v| *v == voice).expect("ranked voices are stock voices");
let refs: Vec<&[f32]> = top.iter().map(|voice| anchors[index(voice)].as_slice()).collect();
spec::lattice(&search::least_squares(&refs, target.as_slice()), 5)
};
let forced = request.accent.family();
let lead = |percents: &[u32]| -> Lead<'static> {
let heaviest = top[percents.iter().enumerate().max_by_key(|(i, p)| (**p, std::cmp::Reverse(*i))).map_or(0, |(i, _)| i)];
match forced {
Some(family) if !heaviest.starts_with(family) => nearest_of(family).map_or(Lead::Heaviest, Lead::Voice),
_ => Lead::Heaviest,
}
};
let reserve = usize::from(forced.is_none() && request.evaluations > 1);
let mut objective = |spec: &str| embed_spoken(&probe, spec, speed.unwrap_or(1.0)).map(|(e, _)| e.similarity(&target));
let mut report =
|e: Evaluated| progress(&MatchProgress::Evaluated { evaluations: e.evaluations, budget: e.budget, best_similarity: e.best });
let mut eval = Evaluator::new(request.evaluations - reserve, &mut objective, &mut report);
let single: Vec<u32> = (0..top.len()).map(|i| if i == 0 { 100 } else { 0 }).collect();
let closest_spec = spec::format(&top, &single, lead(&single));
let closest_similarity = eval.score(&closest_spec)?.unwrap_or(f32::NEG_INFINITY);
let mut best = refine(&top, start, lead, &mut eval)?;
if closest_similarity > best.similarity {
best = search::Found { percents: single, spec: closest_spec, similarity: closest_similarity };
}
if forced.is_none() {
eval.raise_budget(reserve);
let heaviest = best.spec.split(['(', '+']).next().unwrap_or_default();
let other = if heaviest.starts_with('b') { 'a' } else { 'b' };
if let Some(voice) = nearest_of(other) {
let spec = spec::format(&top, &best.percents, Lead::Voice(voice));
if let Some(similarity) = eval.score(&spec)?
&& similarity > best.similarity + 0.001
{
best = search::Found { percents: best.percents, spec, similarity };
}
}
}
Ok(BlendMatch {
spec: best.spec,
similarity: best.similarity,
closest_voice: top[0].to_owned(),
closest_similarity,
ranking: ranking.iter().map(|(voice, similarity)| ((*voice).to_owned(), *similarity)).collect(),
speed,
evaluations: eval.evaluations(),
})
}
fn anchors(
&self,
encoder: &loqui_speaker::SpeakerEncoder,
embed: &mut dyn FnMut(&str) -> Result<SpeakerEmbedding, Error>,
progress: &mut dyn FnMut(&MatchProgress) -> ControlFlow<()>,
) -> Result<Vec<Vec<f32>>, Error> {
let voices = loqui_kokoro::ENGLISH_VOICES;
let kokoro_sha256 = self.inner.tts_variant.map_or("", |variant| variant.file().sha256);
let key = anchors::key(kokoro_sha256, crate::models::SPEAKER_FILE.sha256, voices);
let path = anchors::path(&self.inner.cache, &key);
if let Some(cached) = anchors::read(&path, voices.len(), encoder.dim()) {
return Ok(cached);
}
let mut out = Vec::with_capacity(voices.len());
for (done, voice) in voices.iter().enumerate() {
if progress(&MatchProgress::Preparing { done, total: voices.len() }).is_break() {
return Err(Error::Cancelled);
}
out.push(embed(voice)?.as_slice().to_vec());
}
if let Err(e) = anchors::write(&path, &out) {
tracing::warn!("voice anchors not cached: {e}");
}
Ok(out)
}
}
fn probe_text(transcript: &str) -> String {
let text = transcript.split_whitespace().collect::<Vec<_>>().join(" ");
if text.chars().count() <= *PROBE_CHARS.end() {
return text;
}
let cut: String = text.chars().take(*PROBE_CHARS.end()).collect();
match cut.rfind(['.', '!', '?']) {
Some(end) if end + 1 >= *PROBE_CHARS.start() => cut[..=end].to_owned(),
_ => cut,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{Downloads, SpeakerConfig};
fn quiet(_: &MatchProgress) -> ControlFlow<()> {
ControlFlow::Continue(())
}
#[test]
fn a_bad_request_is_refused_before_anything_loads() {
let cache = std::env::temp_dir().join(format!("loqui-match-refuse-{}", std::process::id()));
let engine =
Engine::builder().cache_dir(&cache).downloads(Downloads::Deny).speaker(Some(SpeakerConfig::default())).build().unwrap();
for (max_voices, evaluations, want) in
[(0, 36, "max_voices"), (7, 36, "max_voices"), (4, 0, "evaluations"), (4, 201, "evaluations")]
{
let mut request = MatchRequest::new(Vec::new());
request.max_voices = max_voices;
request.evaluations = evaluations;
let err = engine.match_voice(&request, &mut quiet).unwrap_err();
assert!(matches!(&err, Error::Invalid(m) if m.contains(want)), "{err:?}");
}
let without_speaker = Engine::builder().cache_dir(&cache).downloads(Downloads::Deny).build().unwrap();
let err = without_speaker.match_voice(&MatchRequest::new(Vec::new()), &mut quiet).unwrap_err();
assert!(matches!(err, Error::Disabled("speaker embeddings")), "{err:?}");
assert!(!cache.exists(), "nothing was fetched or created");
}
#[test]
fn a_long_transcript_is_cut_at_a_sentence() {
let short = "Hello there, this is a short note.";
assert_eq!(probe_text(short), short);
let long = format!("{} And then it went on. {}", "Word ".repeat(30).trim(), "More words ".repeat(20));
let cut = probe_text(&long);
assert!(cut.ends_with("went on."), "{cut}");
assert!(cut.chars().count() <= 200);
let run_on = "word ".repeat(80);
assert_eq!(probe_text(&run_on).chars().count(), 200);
}
}