1use std::fmt;
6
7use serde::{Deserialize, Deserializer, Serialize};
8
9use crate::CoreError;
10
11pub const MAX_SPARSE_CANDIDATES: u32 = 65_536;
13
14#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
16#[serde(transparent)]
17pub struct TokenId(i32);
18
19impl TokenId {
20 pub fn new(raw: i32) -> Result<Self, CoreError> {
26 if raw < 0 {
27 return Err(CoreError::invalid(
28 "token id",
29 "token identifiers must be non-negative",
30 ));
31 }
32 Ok(Self(raw))
33 }
34
35 pub const fn get(self) -> i32 {
37 self.0
38 }
39}
40
41impl TryFrom<i32> for TokenId {
42 type Error = CoreError;
43
44 fn try_from(value: i32) -> Result<Self, Self::Error> {
45 Self::new(value)
46 }
47}
48
49impl From<TokenId> for i32 {
50 fn from(value: TokenId) -> Self {
51 value.get()
52 }
53}
54
55impl<'de> Deserialize<'de> for TokenId {
56 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
57 where
58 D: Deserializer<'de>,
59 {
60 let raw = i32::deserialize(deserializer)?;
61 Self::new(raw).map_err(serde::de::Error::custom)
62 }
63}
64
65impl fmt::Display for TokenId {
66 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
67 self.0.fmt(formatter)
68 }
69}
70
71#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
73#[serde(rename_all = "snake_case", tag = "kind")]
74pub enum CandidateMode {
75 FullVocabulary,
77 Sparse {
79 limit: u32,
81 },
82}
83
84impl CandidateMode {
85 pub fn validate(self) -> Result<(), CoreError> {
91 if let Self::Sparse { limit } = self
92 && !(1..=MAX_SPARSE_CANDIDATES).contains(&limit)
93 {
94 return Err(CoreError::invalid(
95 "candidate mode",
96 format!("sparse limit must be in 1..={MAX_SPARSE_CANDIDATES}"),
97 ));
98 }
99 Ok(())
100 }
101
102 pub const fn sparse_limit(self) -> Option<u32> {
104 match self {
105 Self::FullVocabulary => None,
106 Self::Sparse { limit } => Some(limit),
107 }
108 }
109}
110
111#[cfg(test)]
112mod tests {
113 use super::*;
114
115 #[test]
116 fn token_identifiers_reject_negative_native_values() {
117 assert!(TokenId::new(-1).is_err());
118 assert_eq!(TokenId::new(0).unwrap().get(), 0);
119 assert!(serde_json::from_str::<TokenId>("-1").is_err());
120 assert_eq!(serde_json::from_str::<TokenId>("4").unwrap().get(), 4);
121 }
122
123 #[test]
124 fn sparse_candidate_bounds_are_enforced() {
125 assert!(CandidateMode::Sparse { limit: 0 }.validate().is_err());
126 assert!(
127 CandidateMode::Sparse {
128 limit: MAX_SPARSE_CANDIDATES + 1,
129 }
130 .validate()
131 .is_err()
132 );
133 assert!(CandidateMode::Sparse { limit: 1 }.validate().is_ok());
134 }
135}