modelc 0.1.6

Compile model weight files to standalone executable binaries
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

//! Chat template rendering for transformer models.
//!
//! Reads Jinja2 chat templates from GGUF metadata (`tokenizer.chat_template`) and renders them
//! with a message list before tokenization. Falls back to a simple concatenation when no template
//! is available.

use minijinja::Environment;
use serde::Serialize;

/// A single message in a chat conversation.
#[derive(Serialize, Clone)]
pub struct ChatMessage {
    pub role: String,
    pub content: String,
}

/// Apply a Jinja2 chat template to a list of messages.
///
/// If `template` is `None`, falls back to a simple `role: content\n` concatenation.
pub fn apply_chat_template(
    template: Option<&str>,
    messages: &[ChatMessage],
) -> anyhow::Result<String> {
    if let Some(tpl) = template {
        let mut env = Environment::new();
        env.add_template("chat", tpl)?;
        let tmpl = env.get_template("chat")?;

        let ctx = minijinja::context! {
            messages => messages,
            bos_token => "",
            eos_token => "",
            add_generation_prompt => true,
        };

        Ok(tmpl.render(ctx)?)
    } else {
        Ok(fallback_format(messages))
    }
}

fn fallback_format(messages: &[ChatMessage]) -> String {
    let mut out = String::new();
    for m in messages {
        out.push_str(&m.role);
        out.push_str(": ");
        out.push_str(&m.content);
        out.push('\n');
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn fallback_without_template() {
        let messages = vec![ChatMessage {
            role: "user".to_string(),
            content: "hello".to_string(),
        }];
        let out = apply_chat_template(None, &messages).unwrap();
        assert_eq!(out, "user: hello\n");
    }

    #[test]
    fn simple_jinja_template() {
        let template = "{% for message in messages %}{{ message.role }}: {{ message.content }}\n{% endfor %}assistant:";
        let messages = vec![ChatMessage {
            role: "user".to_string(),
            content: "hi".to_string(),
        }];
        let out = apply_chat_template(Some(template), &messages).unwrap();
        assert_eq!(out, "user: hi\nassistant:");
    }

    #[test]
    fn llama3_like_template() {
        let template = "<|begin_of_text|>{% for message in messages %}<|start_header_id|>{{ message.role }}<|end_header_id|>\n\n{{ message.content }}<|eot_id|>{% endfor %}<|start_header_id|>assistant<|end_header_id|>\n\n";
        let messages = vec![
            ChatMessage {
                role: "system".to_string(),
                content: "You are helpful.".to_string(),
            },
            ChatMessage {
                role: "user".to_string(),
                content: "hello".to_string(),
            },
        ];
        let out = apply_chat_template(Some(template), &messages).unwrap();
        assert!(out.contains("system"));
        assert!(out.contains("user"));
        assert!(out.contains("assistant"));
        assert!(out.contains("<|begin_of_text|>"));
    }
}