use std::fmt;
use serde::{Deserialize, Deserializer, Serialize};
use crate::CoreError;
pub const MAX_SPARSE_CANDIDATES: u32 = 65_536;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
#[serde(transparent)]
pub struct TokenId(i32);
impl TokenId {
pub fn new(raw: i32) -> Result<Self, CoreError> {
if raw < 0 {
return Err(CoreError::invalid(
"token id",
"token identifiers must be non-negative",
));
}
Ok(Self(raw))
}
pub const fn get(self) -> i32 {
self.0
}
}
impl TryFrom<i32> for TokenId {
type Error = CoreError;
fn try_from(value: i32) -> Result<Self, Self::Error> {
Self::new(value)
}
}
impl From<TokenId> for i32 {
fn from(value: TokenId) -> Self {
value.get()
}
}
impl<'de> Deserialize<'de> for TokenId {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let raw = i32::deserialize(deserializer)?;
Self::new(raw).map_err(serde::de::Error::custom)
}
}
impl fmt::Display for TokenId {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(formatter)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "kind")]
pub enum CandidateMode {
FullVocabulary,
Sparse {
limit: u32,
},
}
impl CandidateMode {
pub fn validate(self) -> Result<(), CoreError> {
if let Self::Sparse { limit } = self
&& !(1..=MAX_SPARSE_CANDIDATES).contains(&limit)
{
return Err(CoreError::invalid(
"candidate mode",
format!("sparse limit must be in 1..={MAX_SPARSE_CANDIDATES}"),
));
}
Ok(())
}
pub const fn sparse_limit(self) -> Option<u32> {
match self {
Self::FullVocabulary => None,
Self::Sparse { limit } => Some(limit),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn token_identifiers_reject_negative_native_values() {
assert!(TokenId::new(-1).is_err());
assert_eq!(TokenId::new(0).unwrap().get(), 0);
assert!(serde_json::from_str::<TokenId>("-1").is_err());
assert_eq!(serde_json::from_str::<TokenId>("4").unwrap().get(), 4);
}
#[test]
fn sparse_candidate_bounds_are_enforced() {
assert!(CandidateMode::Sparse { limit: 0 }.validate().is_err());
assert!(
CandidateMode::Sparse {
limit: MAX_SPARSE_CANDIDATES + 1,
}
.validate()
.is_err()
);
assert!(CandidateMode::Sparse { limit: 1 }.validate().is_ok());
}
}