token_terminator/
lib.rs

1//! Rust interoperability helpers for [Token Terminator](https://github.com/AronAxe/Token-Terminator).
2//!
3//! The production Token Terminator runtime remains the Python/Hermes implementation.
4//! This crate intentionally exposes only stable cross-language primitives that Rust
5//! agent adapters can use without embedding Python: content-addressed artifact
6//! identities, digest verification, and the strict character-reduction invariant.
7//!
8//! It is **not** a PyO3 accelerator and does not implement the SQLite vault,
9//! request compiler, tokenizer adapters, or Hermes lifecycle hooks.
10
11#![forbid(unsafe_code)]
12
13use sha2::{Digest, Sha256};
14use std::error::Error;
15use std::fmt;
16
17/// Number of hexadecimal SHA-256 characters used by Token Terminator's normal
18/// short artifact identifier.
19pub const SHORT_ARTIFACT_HEX_LEN: usize = 32;
20
21/// A content identity compatible with Token Terminator's Python artifact vault.
22#[derive(Debug, Clone, PartialEq, Eq)]
23pub struct ArtifactIdentity {
24    /// The normal compact artifact identifier (`a_` plus 32 SHA-256 hex chars).
25    pub artifact_id: String,
26    /// The complete lowercase SHA-256 digest of the UTF-8 content.
27    pub sha256: String,
28}
29
30/// Error returned when a caller supplies something that is not a SHA-256 hex digest.
31#[derive(Debug, Clone, PartialEq, Eq)]
32pub struct InvalidSha256;
33
34impl fmt::Display for InvalidSha256 {
35    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
36        formatter.write_str("expected exactly 64 hexadecimal SHA-256 characters")
37    }
38}
39
40impl Error for InvalidSha256 {}
41
42fn normalize_sha256(value: &str) -> Result<String, InvalidSha256> {
43    if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) {
44        return Err(InvalidSha256);
45    }
46    Ok(value.to_ascii_lowercase())
47}
48
49/// Return the lowercase SHA-256 digest of arbitrary bytes.
50pub fn sha256_hex(bytes: impl AsRef<[u8]>) -> String {
51    let digest = Sha256::digest(bytes.as_ref());
52    format!("{digest:x}")
53}
54
55/// Compute the normal Token Terminator identity for UTF-8 text.
56///
57/// The Python vault hashes the exact UTF-8 bytes and normally identifies the
58/// artifact as `a_` plus the first 32 hexadecimal characters of that digest.
59/// In the astronomically unlikely event of a short-ID collision, the Python
60/// vault falls back to the full digest ID; use [`full_artifact_id_from_sha256`]
61/// when a host needs that collision form explicitly.
62pub fn artifact_identity(content: &str) -> ArtifactIdentity {
63    let sha256 = sha256_hex(content.as_bytes());
64    let artifact_id = format!("a_{}", &sha256[..SHORT_ARTIFACT_HEX_LEN]);
65    ArtifactIdentity {
66        artifact_id,
67        sha256,
68    }
69}
70
71/// Build Token Terminator's normal short artifact ID from a full SHA-256 digest.
72pub fn artifact_id_from_sha256(sha256: &str) -> Result<String, InvalidSha256> {
73    let normalized = normalize_sha256(sha256)?;
74    Ok(format!("a_{}", &normalized[..SHORT_ARTIFACT_HEX_LEN]))
75}
76
77/// Build Token Terminator's full collision-fallback artifact ID from a digest.
78pub fn full_artifact_id_from_sha256(sha256: &str) -> Result<String, InvalidSha256> {
79    let normalized = normalize_sha256(sha256)?;
80    Ok(format!("a_{normalized}"))
81}
82
83/// Verify that UTF-8 text has the expected SHA-256 digest.
84///
85/// Uppercase hexadecimal input is accepted; malformed digests return `false`.
86pub fn verify_sha256(content: &str, expected_sha256: &str) -> bool {
87    normalize_sha256(expected_sha256)
88        .map(|expected| sha256_hex(content.as_bytes()) == expected)
89        .unwrap_or(false)
90}
91
92/// Return whether an artifact ID is compatible with the supplied UTF-8 text.
93///
94/// Both the normal short ID and the full collision-fallback ID are accepted.
95pub fn artifact_id_matches(content: &str, artifact_id: &str) -> bool {
96    let identity = artifact_identity(content);
97    artifact_id == identity.artifact_id || artifact_id == format!("a_{}", identity.sha256)
98}
99
100/// Mirror Token Terminator's character-based strict-reduction invariant.
101///
102/// This counts Unicode scalar values instead of UTF-8 bytes so non-ASCII text
103/// does not appear artificially larger merely because its encoding uses more
104/// than one byte per character. When the Python runtime has an exact tokenizer,
105/// it applies an additional token-level gate after this baseline check.
106pub fn strictly_smaller_chars(raw: &str, candidate: &str) -> bool {
107    candidate.chars().count() < raw.chars().count()
108}
109
110#[cfg(test)]
111mod tests {
112    use super::*;
113
114    const HELLO_SHA256: &str = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824";
115
116    #[test]
117    fn identity_matches_python_layout() {
118        let identity = artifact_identity("hello");
119        assert_eq!(identity.sha256, HELLO_SHA256);
120        assert_eq!(identity.artifact_id, "a_2cf24dba5fb0a30e26e83b2ac5b9e29e");
121    }
122
123    #[test]
124    fn digest_to_short_and_full_ids() {
125        assert_eq!(
126            artifact_id_from_sha256(HELLO_SHA256).unwrap(),
127            "a_2cf24dba5fb0a30e26e83b2ac5b9e29e"
128        );
129        assert_eq!(
130            full_artifact_id_from_sha256(&HELLO_SHA256.to_ascii_uppercase()).unwrap(),
131            format!("a_{HELLO_SHA256}")
132        );
133    }
134
135    #[test]
136    fn malformed_digest_is_rejected() {
137        assert_eq!(artifact_id_from_sha256("nope"), Err(InvalidSha256));
138        assert!(!verify_sha256("hello", "nope"));
139    }
140
141    #[test]
142    fn verification_and_id_matching_work() {
143        assert!(verify_sha256("hello", HELLO_SHA256));
144        assert!(artifact_id_matches(
145            "hello",
146            "a_2cf24dba5fb0a30e26e83b2ac5b9e29e"
147        ));
148        assert!(artifact_id_matches("hello", &format!("a_{HELLO_SHA256}")));
149        assert!(!artifact_id_matches(
150            "goodbye",
151            &format!("a_{HELLO_SHA256}")
152        ));
153    }
154
155    #[test]
156    fn strict_smaller_uses_characters_not_utf8_bytes() {
157        assert!(strictly_smaller_chars("éé", "é"));
158        assert!(!strictly_smaller_chars("é", "é"));
159        assert!(!strictly_smaller_chars("é", "ab"));
160    }
161}
token_terminator/lib.rs

token_terminator/
lib.rs
