Skip to main content

logit_loom_core/
token.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! Token and candidate-exposure contracts.
4
5use std::fmt;
6
7use serde::{Deserialize, Deserializer, Serialize};
8
9use crate::CoreError;
10
11/// Maximum sparse candidates exposed to one transform call.
12pub const MAX_SPARSE_CANDIDATES: u32 = 65_536;
13
14/// A non-negative model-tokenizer identifier.
15#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
16#[serde(transparent)]
17pub struct TokenId(i32);
18
19impl TokenId {
20    /// Creates a token identifier.
21    ///
22    /// # Errors
23    ///
24    /// Returns an error for a negative identifier.
25    pub fn new(raw: i32) -> Result<Self, CoreError> {
26        if raw < 0 {
27            return Err(CoreError::invalid(
28                "token id",
29                "token identifiers must be non-negative",
30            ));
31        }
32        Ok(Self(raw))
33    }
34
35    /// Returns the backend-native integer representation.
36    pub const fn get(self) -> i32 {
37        self.0
38    }
39}
40
41impl TryFrom<i32> for TokenId {
42    type Error = CoreError;
43
44    fn try_from(value: i32) -> Result<Self, Self::Error> {
45        Self::new(value)
46    }
47}
48
49impl From<TokenId> for i32 {
50    fn from(value: TokenId) -> Self {
51        value.get()
52    }
53}
54
55impl<'de> Deserialize<'de> for TokenId {
56    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
57    where
58        D: Deserializer<'de>,
59    {
60        let raw = i32::deserialize(deserializer)?;
61        Self::new(raw).map_err(serde::de::Error::custom)
62    }
63}
64
65impl fmt::Display for TokenId {
66    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
67        self.0.fmt(formatter)
68    }
69}
70
71/// Candidate-logit exposure presented to every stage in a pipeline.
72#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
73#[serde(rename_all = "snake_case", tag = "kind")]
74pub enum CandidateMode {
75    /// Expose every vocabulary token in tokenizer order.
76    FullVocabulary,
77    /// Expose at most the highest-ranked `limit` finite raw logits.
78    Sparse {
79        /// Maximum candidates copied into the transform boundary.
80        limit: u32,
81    },
82}
83
84impl CandidateMode {
85    /// Validates sparse bounds.
86    ///
87    /// # Errors
88    ///
89    /// Returns an error for a zero or excessive sparse limit.
90    pub fn validate(self) -> Result<(), CoreError> {
91        if let Self::Sparse { limit } = self
92            && !(1..=MAX_SPARSE_CANDIDATES).contains(&limit)
93        {
94            return Err(CoreError::invalid(
95                "candidate mode",
96                format!("sparse limit must be in 1..={MAX_SPARSE_CANDIDATES}"),
97            ));
98        }
99        Ok(())
100    }
101
102    /// Returns the sparse bound, or `None` for full-vocabulary exposure.
103    pub const fn sparse_limit(self) -> Option<u32> {
104        match self {
105            Self::FullVocabulary => None,
106            Self::Sparse { limit } => Some(limit),
107        }
108    }
109}
110
111#[cfg(test)]
112mod tests {
113    use super::*;
114
115    #[test]
116    fn token_identifiers_reject_negative_native_values() {
117        assert!(TokenId::new(-1).is_err());
118        assert_eq!(TokenId::new(0).unwrap().get(), 0);
119        assert!(serde_json::from_str::<TokenId>("-1").is_err());
120        assert_eq!(serde_json::from_str::<TokenId>("4").unwrap().get(), 4);
121    }
122
123    #[test]
124    fn sparse_candidate_bounds_are_enforced() {
125        assert!(CandidateMode::Sparse { limit: 0 }.validate().is_err());
126        assert!(
127            CandidateMode::Sparse {
128                limit: MAX_SPARSE_CANDIDATES + 1,
129            }
130            .validate()
131            .is_err()
132        );
133        assert!(CandidateMode::Sparse { limit: 1 }.validate().is_ok());
134    }
135}