use super::*;
#[test]
fn test_long_conversation() {
let template = ChatMLTemplate::new();
let mut messages = vec![ChatMessage::system("System")];
for i in 0..50 {
messages.push(ChatMessage::user(format!("User message {i}")));
messages.push(ChatMessage::assistant(format!("Assistant response {i}")));
}
let result = template.format_conversation(&messages);
assert!(result.is_ok());
let output = result.expect("format failed");
assert!(output.contains("User message 49"));
assert!(output.contains("Assistant response 49"));
}
#[test]
fn test_binary_content_handling() {
let template = ChatMLTemplate::new();
let binary_content = String::from_utf8_lossy(&[0x00, 0x01, 0x02, 0xFF, 0xFE]).to_string();
let messages = vec![ChatMessage::user(&binary_content)];
let result = template.format_conversation(&messages);
assert!(result.is_ok());
}
#[test]
fn test_rtl_text_preserved() {
let template = ChatMLTemplate::new();
let messages = vec![ChatMessage::user("مرحبا بالعالم")]; let result = template.format_conversation(&messages);
assert!(result.is_ok());
let output = result.expect("format failed");
assert!(output.contains("مرحبا بالعالم"));
}
#[test]
fn test_custom_role() {
let template = ChatMLTemplate::new();
let messages = vec![
ChatMessage::new("tool", "Function result: 42"),
ChatMessage::user("What was the result?"),
];
let result = template.format_conversation(&messages);
assert!(result.is_ok());
let output = result.expect("format failed");
assert!(output.contains("tool"));
assert!(output.contains("Function result: 42"));
}
#[test]
fn test_multiline_content() {
let template = ChatMLTemplate::new();
let multiline = "Line 1\nLine 2\nLine 3";
let messages = vec![ChatMessage::user(multiline)];
let result = template.format_conversation(&messages);
assert!(result.is_ok());
let output = result.expect("format failed");
assert!(output.contains("Line 1\nLine 2\nLine 3"));
}
#[test]
fn test_format_creation_idempotent() {
let t1 = create_template(TemplateFormat::ChatML);
let t2 = create_template(TemplateFormat::ChatML);
let messages = vec![ChatMessage::user("Test")];
let o1 = t1.format_conversation(&messages).expect("format1");
let o2 = t2.format_conversation(&messages).expect("format2");
assert_eq!(o1, o2);
}
#[test]
fn test_hf_template_debug() {
let json = r#"{
"chat_template": "{{ message.content }}",
"bos_token": "<s>"
}"#;
let template = HuggingFaceTemplate::from_json(json).expect("parse failed");
let debug_str = format!("{:?}", template);
assert!(debug_str.contains("HuggingFaceTemplate"));
assert!(debug_str.contains("template_str"));
}
#[test]
fn test_detect_format_from_tokens_chatml() {
let tokens = SpecialTokens {
im_start_token: Some("<|im_start|>".to_string()),
im_end_token: Some("<|im_end|>".to_string()),
..Default::default()
};
assert_eq!(detect_format_from_tokens(&tokens), TemplateFormat::ChatML);
}
#[test]
fn test_detect_format_from_tokens_im_start_only() {
let tokens = SpecialTokens {
im_start_token: Some("<|im_start|>".to_string()),
..Default::default()
};
assert_eq!(detect_format_from_tokens(&tokens), TemplateFormat::ChatML);
}
#[test]
fn test_detect_format_from_tokens_llama2() {
let tokens = SpecialTokens {
inst_start: Some("[INST]".to_string()),
inst_end: Some("[/INST]".to_string()),
..Default::default()
};
assert_eq!(detect_format_from_tokens(&tokens), TemplateFormat::Llama2);
}
#[test]
fn test_detect_format_from_tokens_raw() {
let tokens = SpecialTokens::default();
assert_eq!(detect_format_from_tokens(&tokens), TemplateFormat::Raw);
}
#[test]
fn test_llama2_format_message_unknown_role() {
let template = Llama2Template::new();
let result = template.format_message("tool", "tool output");
assert!(result.is_ok());
assert_eq!(result.unwrap(), "tool output");
}
#[test]
fn test_mistral_format_message_system() {
let template = MistralTemplate::new();
let result = template.format_message("system", "system content");
assert!(result.is_ok());
assert_eq!(result.unwrap(), "system content\n\n");
}
#[test]
fn test_mistral_format_message_unknown_role() {
let template = MistralTemplate::new();
let result = template.format_message("tool", "tool output");
assert!(result.is_ok());
assert_eq!(result.unwrap(), "tool output");
}
#[test]
fn test_phi_format_message_all_roles() {
let template = PhiTemplate::new();
let sys = template.format_message("system", "sys").unwrap();
assert_eq!(sys, "sys\n");
let user = template.format_message("user", "usr").unwrap();
assert_eq!(user, "Instruct: usr\n");
let asst = template.format_message("assistant", "asst").unwrap();
assert_eq!(asst, "Output: asst\n");
let unknown = template.format_message("tool", "tool").unwrap();
assert_eq!(unknown, "tool");
}
#[test]
fn test_alpaca_format_message_all_roles() {
let template = AlpacaTemplate::new();
let sys = template.format_message("system", "sys").unwrap();
assert_eq!(sys, "sys\n\n");
let user = template.format_message("user", "usr").unwrap();
assert_eq!(user, "### Instruction:\nusr\n\n");
let asst = template.format_message("assistant", "asst").unwrap();
assert_eq!(asst, "### Response:\nasst\n\n");
let unknown = template.format_message("tool", "tool").unwrap();
assert_eq!(unknown, "tool");
}
#[test]
fn test_raw_format_message() {
let template = RawTemplate::new();
let result = template.format_message("any_role", "content");
assert!(result.is_ok());
assert_eq!(result.unwrap(), "content");
}
#[test]
fn test_chatml_clone() {
let template = ChatMLTemplate::new();
let cloned = template.clone();
let messages = vec![ChatMessage::user("test")];
assert_eq!(
template.format_conversation(&messages).unwrap(),
cloned.format_conversation(&messages).unwrap()
);
}
#[test]
fn test_llama2_clone() {
let template = Llama2Template::new();
let cloned = template.clone();
let messages = vec![ChatMessage::user("test")];
assert_eq!(
template.format_conversation(&messages).unwrap(),
cloned.format_conversation(&messages).unwrap()
);
}
#[test]
fn test_mistral_clone() {
let template = MistralTemplate::new();
let cloned = template.clone();
let messages = vec![ChatMessage::user("test")];
assert_eq!(
template.format_conversation(&messages).unwrap(),
cloned.format_conversation(&messages).unwrap()
);
}
#[test]
fn test_phi_clone() {
let template = PhiTemplate::new();
let cloned = template.clone();
let messages = vec![ChatMessage::user("test")];
assert_eq!(
template.format_conversation(&messages).unwrap(),
cloned.format_conversation(&messages).unwrap()
);
}
#[test]
fn test_alpaca_clone() {
let template = AlpacaTemplate::new();
let cloned = template.clone();
let messages = vec![ChatMessage::user("test")];
assert_eq!(
template.format_conversation(&messages).unwrap(),
cloned.format_conversation(&messages).unwrap()
);
}
#[test]
fn test_raw_clone() {
let template = RawTemplate::new();
let cloned = template.clone();
let messages = vec![ChatMessage::user("test")];
assert_eq!(
template.format_conversation(&messages).unwrap(),
cloned.format_conversation(&messages).unwrap()
);
}
#[test]
fn test_chatml_with_custom_tokens() {
let tokens = SpecialTokens {
bos_token: Some("<bos>".to_string()),
eos_token: Some("<eos>".to_string()),
im_start_token: Some("<start>".to_string()),
im_end_token: Some("<end>".to_string()),
..Default::default()
};
let template = ChatMLTemplate::with_tokens(tokens);
assert_eq!(
template.special_tokens().bos_token,
Some("<bos>".to_string())
);
}
#[test]
fn test_template_format_debug() {
let formats = [
TemplateFormat::ChatML,
TemplateFormat::Llama2,
TemplateFormat::Mistral,
TemplateFormat::Phi,
TemplateFormat::Alpaca,
TemplateFormat::Custom,
TemplateFormat::Raw,
];
for fmt in &formats {
let debug = format!("{:?}", fmt);
assert!(!debug.is_empty());
}
}
#[test]
fn test_template_format_copy() {
let fmt = TemplateFormat::ChatML;
let copied = fmt;
assert_eq!(fmt, copied);
}
#[test]
fn test_special_tokens_debug_clone() {
let tokens = SpecialTokens {
bos_token: Some("<s>".to_string()),
..Default::default()
};
let cloned = tokens.clone();
assert_eq!(tokens.bos_token, cloned.bos_token);
let debug = format!("{:?}", tokens);
assert!(debug.contains("bos_token"));
}
#[test]
fn test_chat_message_debug_eq() {
let msg1 = ChatMessage::user("hello");
let msg2 = ChatMessage::user("hello");
let msg3 = ChatMessage::user("world");
assert_eq!(msg1, msg2);
assert_ne!(msg1, msg3);
let debug = format!("{:?}", msg1);
assert!(debug.contains("user"));
}
#[test]
fn test_hf_template_format_message() {
let json = r#"{
"chat_template": "{% for message in messages %}{{ message.content }}{% endfor %}",
"bos_token": "<s>"
}"#;
let template = HuggingFaceTemplate::from_json(json).expect("parse failed");
let result = template.format_message("user", "hello");
assert!(result.is_ok());
}
#[test]
fn test_hf_template_accessors() {
let json = r#"{
"chat_template": "<|im_start|>{{ message.role }}",
"bos_token": "<s>"
}"#;
let template = HuggingFaceTemplate::from_json(json).expect("parse failed");
assert_eq!(template.format(), TemplateFormat::ChatML);
assert!(template.supports_system_prompt());
}
#[test]
fn test_hf_template_alpaca_detection() {
let json = r####"{
"chat_template": "### Instruction: {{ message.content }}",
"bos_token": "<s>"
}"####;
let template = HuggingFaceTemplate::from_json(json).expect("parse failed");
assert_eq!(template.format(), TemplateFormat::Alpaca);
}
#[test]
fn test_create_template_custom() {
let template = create_template(TemplateFormat::Custom);
assert_eq!(template.format(), TemplateFormat::Raw); }
#[test]
fn test_chatml_format_message() {
let template = ChatMLTemplate::new();
let result = template.format_message("user", "hello");
assert!(result.is_ok());
assert_eq!(result.unwrap(), "<|im_start|>user\nhello<|im_end|>\n");
}
#[test]
fn test_llama2_system_merged() {
let template = Llama2Template::new();
let messages = vec![
ChatMessage::system("Be helpful"),
ChatMessage::user("Hello"),
];
let result = template.format_conversation(&messages);
assert!(result.is_ok());
let output = result.unwrap();
assert!(output.contains("<<SYS>>"));
assert!(output.contains("Be helpful"));
assert!(output.contains("[INST]"));
}
#[test]
fn test_llama2_multi_turn_no_system() {
let template = Llama2Template::new();
let messages = vec![
ChatMessage::user("First"),
ChatMessage::assistant("Response 1"),
ChatMessage::user("Second"),
];
let result = template.format_conversation(&messages);
assert!(result.is_ok());
let output = result.unwrap();
assert!(!output.contains("<<SYS>>"));
assert!(output.contains("First"));
assert!(output.contains("Response 1"));
assert!(output.contains("Second"));
}