use std::path::Path;
use anyhow::{anyhow, Context, Result};
const SYSTEM_IMAGE: &str =
"You are a helpful assistant who will generate images from a give prompt.";
const SYSTEM_VIDEO: &str =
"You are a helpful assistant who will generate videos from a give prompt.";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Template {
Qwen,
Nemotron,
}
pub struct Tokenizer {
tok: tokenizers::Tokenizer,
pub template: Template,
pub use_system_prompt: bool,
pub eos: u32,
pub vision_start: u32,
}
pub struct PromptSpec {
pub prompt: String,
pub negative: bool,
pub n_frames: i64,
pub fps: f32,
pub width: i64,
pub height: i64,
}
impl Tokenizer {
pub fn load(path: &Path, template: Template, use_system_prompt: bool) -> Result<Tokenizer> {
let tok = tokenizers::Tokenizer::from_file(path)
.map_err(|e| anyhow!("{}: {e}", path.display()))?;
let id = |s: &str| {
tok.token_to_id(s)
.ok_or_else(|| anyhow!("{}: no {s} token", path.display()))
};
Ok(Tokenizer {
eos: id("<|im_end|>")?,
vision_start: id("<|vision_start|>")?,
tok,
template,
use_system_prompt,
})
}
pub fn user_text(spec: &PromptSpec) -> String {
let is_image = spec.n_frames == 1;
let mut text = spec.prompt.clone();
let mut append = |s: String| {
let base = text.trim_end_matches('.');
text = if base.is_empty() {
s
} else {
format!("{base}. {s}")
};
};
if !is_image {
let duration = spec.n_frames as f64 / spec.fps as f64;
append(if spec.negative {
format!(
"The video is not {duration:.1} seconds long and is not of {:.0} FPS.",
spec.fps
)
} else {
format!(
"The video is {duration:.1} seconds long and is of {:.0} FPS.",
spec.fps
)
});
}
let (h, w) = (spec.height, spec.width);
append(match (is_image, spec.negative) {
(true, false) => format!("This image is of {h}x{w} resolution."),
(true, true) => format!("This image is not of {h}x{w} resolution."),
(false, false) => format!("This video is of {h}x{w} resolution."),
(false, true) => format!("This video is not of {h}x{w} resolution."),
});
text
}
pub fn encode(&self, spec: &PromptSpec) -> Result<Vec<u32>> {
let text = render(self.template, self.use_system_prompt, spec);
self.encode_text(&text)
}
pub fn encode_chat(&self, user: &str, system: Option<&str>) -> Result<Vec<u32>> {
let text = render_chat(self.template, system, user);
self.encode_text(&text)
}
fn encode_text(&self, text: &str) -> Result<Vec<u32>> {
let enc = self
.tok
.encode(text, false)
.map_err(|e| anyhow!("tokenizing the prompt: {e}"))
.context("cosmos3 tokenizer")?;
let mut ids = enc.get_ids().to_vec();
ids.push(self.eos);
ids.push(self.vision_start);
Ok(ids)
}
}
pub fn render(template: Template, use_system_prompt: bool, spec: &PromptSpec) -> String {
let user = Tokenizer::user_text(spec);
let system = use_system_prompt.then_some(if spec.n_frames == 1 {
SYSTEM_IMAGE
} else {
SYSTEM_VIDEO
});
render_chat(template, system, &user)
}
fn render_chat(template: Template, system: Option<&str>, user: &str) -> String {
match template {
Template::Qwen => {
let mut s = String::new();
if let Some(sys) = system {
s.push_str(&format!("<|im_start|>system\n{sys}<|im_end|>\n"));
}
s.push_str(&format!(
"<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n"
));
s
}
Template::Nemotron => format!(
"\n<|im_start|>system\n{}<|im_end|>\n<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n<think>\n",
system.unwrap_or("")
),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn user_text_templates() {
let spec = PromptSpec {
prompt: "A cat sits on a mat.".into(),
negative: false,
n_frames: 49,
fps: 24.0,
width: 832,
height: 480,
};
assert_eq!(
Tokenizer::user_text(&spec),
"A cat sits on a mat. The video is 2.0 seconds long and is of 24 FPS. This video is of 480x832 resolution."
);
let img = PromptSpec {
n_frames: 1,
negative: true,
prompt: String::new(),
..spec
};
assert_eq!(
Tokenizer::user_text(&img),
"This image is not of 480x832 resolution."
);
}
#[test]
fn chat_templates() {
let spec = PromptSpec {
prompt: "A cat.".into(),
negative: false,
n_frames: 1,
fps: 24.0,
width: 256,
height: 256,
};
let user = "A cat. This image is of 256x256 resolution.";
assert_eq!(
render(Template::Qwen, true, &spec),
format!("<|im_start|>system\n{SYSTEM_IMAGE}<|im_end|>\n<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n")
);
assert_eq!(
render(Template::Qwen, false, &spec),
format!("<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n")
);
assert_eq!(
render(Template::Nemotron, false, &spec),
format!("\n<|im_start|>system\n<|im_end|>\n<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n<think>\n")
);
}
}