llm-tokenizer 0.0.1

Tokenizer utilities for LLM inference
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

//! Tokenizer utilities for LLM inference
//!
//! This crate is under development as part of the Shepherd project.

/// Placeholder for future implementation
pub fn version() -> &'static str {
    env!("CARGO_PKG_VERSION")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_version() {
        assert_eq!(version(), "0.0.1");
    }
}