use super::model::{CanaryHParams, Vocab};
const FORMAT: &str = "canary2";
const SP_SPACE: &str = "\u{2581}";
const EMO_UNDEFINED: &str = "<|emo:undefined|>";
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PromptError {
UnsupportedLanguage(String),
MissingSpecial(String),
MissingPiece(String),
UnsupportedFormat(String),
}
impl std::fmt::Display for PromptError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::UnsupportedLanguage(lang) => {
write!(
f,
"language {lang:?} is not in the model's general.languages"
)
}
Self::MissingSpecial(name) => {
write!(f, "the model has no stt.canary.special.{name}_id")
}
Self::MissingPiece(piece) => {
write!(f, "the vocabulary has no piece {piece:?}")
}
Self::UnsupportedFormat(format) => {
write!(
f,
"prompt format {format:?} is not supported, only {FORMAT:?} is"
)
}
}
}
}
impl std::error::Error for PromptError {}
pub fn build(
hp: &CanaryHParams,
vocab: &Vocab,
lang: &str,
pnc: bool,
) -> Result<Vec<i32>, PromptError> {
if hp.prompt_format != FORMAT {
return Err(PromptError::UnsupportedFormat(hp.prompt_format.clone()));
}
if !hp.languages.iter().any(|l| l == lang) {
return Err(PromptError::UnsupportedLanguage(lang.to_owned()));
}
let special = |name: &str| {
hp.special
.get(name)
.map(|&id| id as i32)
.ok_or_else(|| PromptError::MissingSpecial(name.to_owned()))
};
let piece = |text: &str| {
vocab
.id_of(text)
.map(|id| id as i32)
.ok_or_else(|| PromptError::MissingPiece(text.to_owned()))
};
let lang_id = special(&format!("lang.{lang}"))?;
let mut ids = Vec::with_capacity(10);
if hp.single_sp {
ids.push(piece(SP_SPACE)?);
}
ids.push(special("startofcontext")?);
ids.push(special("startoftranscript")?);
ids.push(piece(EMO_UNDEFINED)?);
ids.push(lang_id);
ids.push(lang_id);
ids.push(special(if pnc { "pnc" } else { "nopnc" })?);
ids.push(special("noitn")?);
ids.push(special("notimestamp")?);
ids.push(special("nodiarize")?);
Ok(ids)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::speech::model::phase1_hparams;
fn hparams() -> CanaryHParams {
let mut hp = phase1_hparams();
hp.special.extend(
[
("pnc", 5),
("nopnc", 6),
("noitn", 9),
("notimestamp", 11),
("nodiarize", 13),
]
.into_iter()
.map(|(k, v)| (k.to_string(), v)),
);
hp
}
fn vocab() -> Vocab {
let mut pieces: Vec<String> = (0..20).map(|i| format!("<|special{i}|>")).collect();
pieces[0] = "<unk>".into();
pieces[2] = "<pad>".into();
pieces.extend(
[
"\u{2581}",
"<|emo:undefined|>",
"\u{2581}Hello",
",",
"\u{2581}world",
".",
]
.map(str::to_owned),
);
Vocab { pieces }
}
#[test]
fn build_renders_ten_ids_on_a_single_sp_vocabulary() {
let ids = build(&hparams(), &vocab(), "en", true).unwrap();
assert_eq!(ids, [20, 7, 4, 21, 64, 64, 5, 9, 11, 13]);
}
#[test]
fn build_fills_both_language_slots_with_the_requested_language() {
let mut hp = hparams();
hp.special.insert("lang.de".to_string(), 78);
let ids = build(&hp, &vocab(), "de", true).unwrap();
assert_eq!(ids, [20, 7, 4, 21, 78, 78, 5, 9, 11, 13]);
}
#[test]
fn build_drops_the_context_slot_on_an_aggregate_vocabulary() {
let mut hp = hparams();
hp.single_sp = false;
let ids = build(&hp, &vocab(), "en", true).unwrap();
assert_eq!(ids, [7, 4, 21, 64, 64, 5, 9, 11, 13]);
}
#[test]
fn build_uses_nopnc_when_punctuation_is_off() {
let ids = build(&hparams(), &vocab(), "en", false).unwrap();
assert_eq!(ids[6], 6);
assert_eq!(ids.len(), 10);
}
#[test]
fn build_rejects_a_language_outside_general_languages() {
assert_eq!(
build(&hparams(), &vocab(), "xx", true),
Err(PromptError::UnsupportedLanguage("xx".into()))
);
}
#[test]
fn build_rejects_a_listed_language_without_a_special_id() {
assert_eq!(
build(&hparams(), &vocab(), "de", true),
Err(PromptError::MissingSpecial("lang.de".into()))
);
}
#[test]
fn build_rejects_a_missing_piece_and_another_format() {
let mut vocab = vocab();
vocab.pieces.truncate(21);
assert_eq!(
build(&hparams(), &vocab, "en", true),
Err(PromptError::MissingPiece("<|emo:undefined|>".into()))
);
let mut hp = hparams();
hp.prompt_format = "canary1".into();
assert_eq!(
build(&hp, &vocab, "en", true),
Err(PromptError::UnsupportedFormat("canary1".into()))
);
}
#[test]
fn detokenize_drops_specials_maps_the_marker_and_trims() {
let v = vocab();
let ids = [4, 22, 23, 0, 24, 2, 25, 3, 99, -1];
assert_eq!(v.detokenize(&ids), "Hello, world.");
assert_eq!(v.detokenize(&[20, 20]), "");
assert_eq!(v.detokenize(&[]), "");
}
#[test]
fn prompt_error_displays_its_cause() {
assert_eq!(
PromptError::UnsupportedLanguage("xx".into()).to_string(),
"language \"xx\" is not in the model's general.languages"
);
}
}