/// Marks the boundary between separate documents in a training corpus
pub const END_OF_TEXT: &str = "<|endoftext|>";
/// Inserted when the input sequence is shorter than the expected length
pub const PADDING: &str = "<|pad|>";
/// Replaces any token that is not present in the vocabulary
pub const UNKNOWN: &str = "<|unk|>";
/// Marks the beginning of a sequence
pub const BEGIN_OF_SEQUENCE: &str = "<|bos|>";
/// Marks the end of a sequence
pub const END_OF_SEQUENCE: &str = "<|eos|>";
/// Separates two segments within the same input (e.g. question and context)
pub const SEPARATOR: &str = "<|sep|>";
/// Placeholder used during masked language modelling
pub const MASK: &str = "<|mask|>";