splintr 0.15.0

// =============================================================================
// AUTO-GENERATED FILE - DO NOT EDIT MANUALLY
// Generated by: scripts/generate_agent_tokens.py --lang rust
// Source of truth: AGENT_TOKENS and MODELS in that script
// =============================================================================
//
// Compile-time agent-token ids, one module per bundled vocabulary.
// Whisper has no module: it carries no agent tokens.
//
// Included by agent_tokens.rs, which holds the tests for these ids.

/// Agent tokens for cl100k_base (GPT-4, GPT-3.5-turbo).
///
/// The 54-token block starts at 100277. Every id here is also reachable
/// at runtime as `from_pretrained("cl100k_base")?.special_token_id(name)`;
/// these constants are the compile-time form, so a typo is a compile
/// error rather than a `None` to unwrap.
pub mod cl100k_agent_tokens {
    // Conversation & Roles
    /// System role - system instructions
    pub const SYSTEM: u32 = 100277;
    /// User role - user input
    pub const USER: u32 = 100278;
    /// Assistant role - model output
    pub const ASSISTANT: u32 = 100279;
    /// Start of message - ChatML wrapper
    pub const IM_START: u32 = 100280;
    /// End of message - ChatML wrapper
    pub const IM_END: u32 = 100281;

    // Reasoning/Thinking
    /// Start of thinking - Chain-of-Thought
    pub const THINK: u32 = 100282;
    /// End of thinking
    pub const THINK_END: u32 = 100283;

    // ReAct Agent Loop
    /// Start of plan - action planning
    pub const PLAN: u32 = 100284;
    /// End of plan
    pub const PLAN_END: u32 = 100285;
    /// Start of step - individual action step
    pub const STEP: u32 = 100286;
    /// End of step
    pub const STEP_END: u32 = 100287;
    /// Start of action - agent action
    pub const ACT: u32 = 100288;
    /// End of action
    pub const ACT_END: u32 = 100289;
    /// Start of observation - environment feedback
    pub const OBSERVE: u32 = 100290;
    /// End of observation
    pub const OBSERVE_END: u32 = 100291;

    // Tool/Function Calling
    /// Start of function call - function invocation
    pub const FUNCTION: u32 = 100292;
    /// End of function call
    pub const FUNCTION_END: u32 = 100293;
    /// Start of function result - return value
    pub const RESULT: u32 = 100294;
    /// End of function result
    pub const RESULT_END: u32 = 100295;
    /// Start of error - error message
    pub const ERROR: u32 = 100296;
    /// End of error
    pub const ERROR_END: u32 = 100297;

    // Code Execution
    /// Start of code - inline code execution
    pub const CODE: u32 = 100298;
    /// End of code
    pub const CODE_END: u32 = 100299;
    /// Start of output - execution output
    pub const OUTPUT: u32 = 100300;
    /// End of output
    pub const OUTPUT_END: u32 = 100301;
    /// Start of language tag - code language
    pub const LANG: u32 = 100302;
    /// End of language tag
    pub const LANG_END: u32 = 100303;

    // RAG & Citations
    /// Start of context - retrieved context
    pub const CONTEXT: u32 = 100304;
    /// End of context
    pub const CONTEXT_END: u32 = 100305;
    /// Start of quote - exact citation
    pub const QUOTE: u32 = 100306;
    /// End of quote
    pub const QUOTE_END: u32 = 100307;
    /// Start of cite - citation reference
    pub const CITE: u32 = 100308;
    /// End of cite
    pub const CITE_END: u32 = 100309;
    /// Start of source - document source
    pub const SOURCE: u32 = 100310;
    /// End of source
    pub const SOURCE_END: u32 = 100311;

    // Memory/State Management
    /// Start of memory - persistent memory
    pub const MEMORY: u32 = 100312;
    /// End of memory
    pub const MEMORY_END: u32 = 100313;
    /// Start of recall - memory retrieval
    pub const RECALL: u32 = 100314;
    /// End of recall
    pub const RECALL_END: u32 = 100315;

    // Control Tokens
    /// Padding token
    pub const PAD: u32 = 100316;
    /// Stop generation token
    pub const STOP: u32 = 100317;
    /// Separator token
    pub const SEP: u32 = 100318;

    // Multimodal Placeholders
    /// Start of image - image placeholder
    pub const IMAGE: u32 = 100319;
    /// End of image
    pub const IMAGE_END: u32 = 100320;
    /// Start of audio - audio placeholder
    pub const AUDIO: u32 = 100321;
    /// End of audio
    pub const AUDIO_END: u32 = 100322;
    /// Start of video - video placeholder
    pub const VIDEO: u32 = 100323;
    /// End of video
    pub const VIDEO_END: u32 = 100324;

    // Document Structure
    /// Start of title - document/section title
    pub const TITLE: u32 = 100325;
    /// End of title
    pub const TITLE_END: u32 = 100326;
    /// Start of section - semantic document section
    pub const SECTION: u32 = 100327;
    /// End of section
    pub const SECTION_END: u32 = 100328;
    /// Start of summary - condensed content summary
    pub const SUMMARY: u32 = 100329;
    /// End of summary
    pub const SUMMARY_END: u32 = 100330;
}

/// Agent tokens for o200k_base (GPT-4o).
///
/// The 54-token block starts at 200019. Every id here is also reachable
/// at runtime as `from_pretrained("o200k_base")?.special_token_id(name)`;
/// these constants are the compile-time form, so a typo is a compile
/// error rather than a `None` to unwrap.
pub mod o200k_agent_tokens {
    // Conversation & Roles
    /// System role - system instructions
    pub const SYSTEM: u32 = 200019;
    /// User role - user input
    pub const USER: u32 = 200020;
    /// Assistant role - model output
    pub const ASSISTANT: u32 = 200021;
    /// Start of message - ChatML wrapper
    pub const IM_START: u32 = 200022;
    /// End of message - ChatML wrapper
    pub const IM_END: u32 = 200023;

    // Reasoning/Thinking
    /// Start of thinking - Chain-of-Thought
    pub const THINK: u32 = 200024;
    /// End of thinking
    pub const THINK_END: u32 = 200025;

    // ReAct Agent Loop
    /// Start of plan - action planning
    pub const PLAN: u32 = 200026;
    /// End of plan
    pub const PLAN_END: u32 = 200027;
    /// Start of step - individual action step
    pub const STEP: u32 = 200028;
    /// End of step
    pub const STEP_END: u32 = 200029;
    /// Start of action - agent action
    pub const ACT: u32 = 200030;
    /// End of action
    pub const ACT_END: u32 = 200031;
    /// Start of observation - environment feedback
    pub const OBSERVE: u32 = 200032;
    /// End of observation
    pub const OBSERVE_END: u32 = 200033;

    // Tool/Function Calling
    /// Start of function call - function invocation
    pub const FUNCTION: u32 = 200034;
    /// End of function call
    pub const FUNCTION_END: u32 = 200035;
    /// Start of function result - return value
    pub const RESULT: u32 = 200036;
    /// End of function result
    pub const RESULT_END: u32 = 200037;
    /// Start of error - error message
    pub const ERROR: u32 = 200038;
    /// End of error
    pub const ERROR_END: u32 = 200039;

    // Code Execution
    /// Start of code - inline code execution
    pub const CODE: u32 = 200040;
    /// End of code
    pub const CODE_END: u32 = 200041;
    /// Start of output - execution output
    pub const OUTPUT: u32 = 200042;
    /// End of output
    pub const OUTPUT_END: u32 = 200043;
    /// Start of language tag - code language
    pub const LANG: u32 = 200044;
    /// End of language tag
    pub const LANG_END: u32 = 200045;

    // RAG & Citations
    /// Start of context - retrieved context
    pub const CONTEXT: u32 = 200046;
    /// End of context
    pub const CONTEXT_END: u32 = 200047;
    /// Start of quote - exact citation
    pub const QUOTE: u32 = 200048;
    /// End of quote
    pub const QUOTE_END: u32 = 200049;
    /// Start of cite - citation reference
    pub const CITE: u32 = 200050;
    /// End of cite
    pub const CITE_END: u32 = 200051;
    /// Start of source - document source
    pub const SOURCE: u32 = 200052;
    /// End of source
    pub const SOURCE_END: u32 = 200053;

    // Memory/State Management
    /// Start of memory - persistent memory
    pub const MEMORY: u32 = 200054;
    /// End of memory
    pub const MEMORY_END: u32 = 200055;
    /// Start of recall - memory retrieval
    pub const RECALL: u32 = 200056;
    /// End of recall
    pub const RECALL_END: u32 = 200057;

    // Control Tokens
    /// Padding token
    pub const PAD: u32 = 200058;
    /// Stop generation token
    pub const STOP: u32 = 200059;
    /// Separator token
    pub const SEP: u32 = 200060;

    // Multimodal Placeholders
    /// Start of image - image placeholder
    pub const IMAGE: u32 = 200061;
    /// End of image
    pub const IMAGE_END: u32 = 200062;
    /// Start of audio - audio placeholder
    pub const AUDIO: u32 = 200063;
    /// End of audio
    pub const AUDIO_END: u32 = 200064;
    /// Start of video - video placeholder
    pub const VIDEO: u32 = 200065;
    /// End of video
    pub const VIDEO_END: u32 = 200066;

    // Document Structure
    /// Start of title - document/section title
    pub const TITLE: u32 = 200067;
    /// End of title
    pub const TITLE_END: u32 = 200068;
    /// Start of section - semantic document section
    pub const SECTION: u32 = 200069;
    /// End of section
    pub const SECTION_END: u32 = 200070;
    /// Start of summary - condensed content summary
    pub const SUMMARY: u32 = 200071;
    /// End of summary
    pub const SUMMARY_END: u32 = 200072;
}

/// Agent tokens for Llama 3 family.
///
/// The 54-token block starts at 128300. Every id here is also reachable
/// at runtime as `from_pretrained("llama3")?.special_token_id(name)`;
/// these constants are the compile-time form, so a typo is a compile
/// error rather than a `None` to unwrap.
///
/// Llama 3 family defines AUDIO, AUDIO_END, IMAGE, IMAGE_END, VIDEO, VIDEO_END itself, so those constants carry the
/// vocabulary's own id — below `base_vocab_size`, the id the checkpoint
/// was trained on — rather than a splintr-appended one.
///
/// The vocabulary's own markers are included too, so this module is its
/// whole special-token surface. A native name that would collide with
/// an agent token of the same meaning takes a `_NATIVE` suffix.
pub mod llama3_agent_tokens {
    // Vocabulary-native tokens
    /// Start of text marker
    pub const BEGIN_OF_TEXT: u32 = 128000;
    /// End of text marker
    pub const END_OF_TEXT: u32 = 128001;
    /// Padding token for fine-tuning - added in Llama 3.1
    pub const FINETUNE_RIGHT_PAD_ID: u32 = 128004;
    /// Step marker for reasoning - added in Llama 3.2-Vision
    pub const STEP_ID: u32 = 128005;
    /// Start of header marker
    pub const START_HEADER_ID: u32 = 128006;
    /// End of header marker
    pub const END_HEADER_ID: u32 = 128007;
    /// End of message marker for tool use - added in Llama 3.1
    pub const EOM_ID: u32 = 128008;
    /// End of turn marker
    pub const EOT_ID: u32 = 128009;
    /// Python code interpreter marker - added in Llama 3.1
    pub const PYTHON_TAG: u32 = 128010;
    /// Image placeholder - Llama 3.2-Vision official
    pub const IMAGE: u32 = 128256;
    /// End of image placeholder
    pub const IMAGE_END: u32 = 128257;
    /// Audio placeholder
    pub const AUDIO: u32 = 128258;
    /// End of audio placeholder
    pub const AUDIO_END: u32 = 128259;
    /// Video placeholder
    pub const VIDEO: u32 = 128260;
    /// End of video placeholder
    pub const VIDEO_END: u32 = 128261;

    // Conversation & Roles
    /// System role - system instructions
    pub const SYSTEM: u32 = 128300;
    /// User role - user input
    pub const USER: u32 = 128301;
    /// Assistant role - model output
    pub const ASSISTANT: u32 = 128302;
    /// Start of message - ChatML wrapper
    pub const IM_START: u32 = 128303;
    /// End of message - ChatML wrapper
    pub const IM_END: u32 = 128304;

    // Reasoning/Thinking
    /// Start of thinking - Chain-of-Thought
    pub const THINK: u32 = 128305;
    /// End of thinking
    pub const THINK_END: u32 = 128306;

    // ReAct Agent Loop
    /// Start of plan - action planning
    pub const PLAN: u32 = 128307;
    /// End of plan
    pub const PLAN_END: u32 = 128308;
    /// Start of step - individual action step
    pub const STEP: u32 = 128309;
    /// End of step
    pub const STEP_END: u32 = 128310;
    /// Start of action - agent action
    pub const ACT: u32 = 128311;
    /// End of action
    pub const ACT_END: u32 = 128312;
    /// Start of observation - environment feedback
    pub const OBSERVE: u32 = 128313;
    /// End of observation
    pub const OBSERVE_END: u32 = 128314;

    // Tool/Function Calling
    /// Start of function call - function invocation
    pub const FUNCTION: u32 = 128315;
    /// End of function call
    pub const FUNCTION_END: u32 = 128316;
    /// Start of function result - return value
    pub const RESULT: u32 = 128317;
    /// End of function result
    pub const RESULT_END: u32 = 128318;
    /// Start of error - error message
    pub const ERROR: u32 = 128319;
    /// End of error
    pub const ERROR_END: u32 = 128320;

    // Code Execution
    /// Start of code - inline code execution
    pub const CODE: u32 = 128321;
    /// End of code
    pub const CODE_END: u32 = 128322;
    /// Start of output - execution output
    pub const OUTPUT: u32 = 128323;
    /// End of output
    pub const OUTPUT_END: u32 = 128324;
    /// Start of language tag - code language
    pub const LANG: u32 = 128325;
    /// End of language tag
    pub const LANG_END: u32 = 128326;

    // RAG & Citations
    /// Start of context - retrieved context
    pub const CONTEXT: u32 = 128327;
    /// End of context
    pub const CONTEXT_END: u32 = 128328;
    /// Start of quote - exact citation
    pub const QUOTE: u32 = 128329;
    /// End of quote
    pub const QUOTE_END: u32 = 128330;
    /// Start of cite - citation reference
    pub const CITE: u32 = 128331;
    /// End of cite
    pub const CITE_END: u32 = 128332;
    /// Start of source - document source
    pub const SOURCE: u32 = 128333;
    /// End of source
    pub const SOURCE_END: u32 = 128334;

    // Memory/State Management
    /// Start of memory - persistent memory
    pub const MEMORY: u32 = 128335;
    /// End of memory
    pub const MEMORY_END: u32 = 128336;
    /// Start of recall - memory retrieval
    pub const RECALL: u32 = 128337;
    /// End of recall
    pub const RECALL_END: u32 = 128338;

    // Control Tokens
    /// Padding token
    pub const PAD: u32 = 128339;
    /// Stop generation token
    pub const STOP: u32 = 128340;
    /// Separator token
    pub const SEP: u32 = 128341;

    // Document Structure
    /// Start of title - document/section title
    pub const TITLE: u32 = 128348;
    /// End of title
    pub const TITLE_END: u32 = 128349;
    /// Start of section - semantic document section
    pub const SECTION: u32 = 128350;
    /// End of section
    pub const SECTION_END: u32 = 128351;
    /// Start of summary - condensed content summary
    pub const SUMMARY: u32 = 128352;
    /// End of summary
    pub const SUMMARY_END: u32 = 128353;
}

/// Agent tokens for DeepSeek V3/R1.
///
/// The 54-token block starts at 128900. Every id here is also reachable
/// at runtime as `from_pretrained("deepseek_v3")?.special_token_id(name)`;
/// these constants are the compile-time form, so a typo is a compile
/// error rather than a `None` to unwrap.
///
/// The vocabulary's own markers are included too, so this module is its
/// whole special-token surface. A native name that would collide with
/// an agent token of the same meaning takes a `_NATIVE` suffix.
pub mod deepseek_v3_agent_tokens {
    // Vocabulary-native tokens
    /// Begin of sentence marker
    pub const BEGIN_OF_SENTENCE: u32 = 0;
    /// End of sentence marker
    pub const END_OF_SENTENCE: u32 = 1;
    /// Native padding token
    pub const PAD_NATIVE: u32 = 2;
    /// Native `<think>` token
    pub const THINK_NATIVE: u32 = 128798;
    /// Native `</think>` token
    pub const THINK_END_NATIVE: u32 = 128799;
    /// Fill-in-the-Middle hole
    pub const FIM_HOLE: u32 = 128800;
    /// Fill-in-the-Middle begin
    pub const FIM_BEGIN: u32 = 128801;
    /// Fill-in-the-Middle end
    pub const FIM_END: u32 = 128802;
    /// Native `<｜User｜>` token
    pub const USER_NATIVE: u32 = 128803;
    /// Native `<｜Assistant｜>` token
    pub const ASSISTANT_NATIVE: u32 = 128804;
    /// End of turn `<|EOT|>`
    pub const EOT: u32 = 128805;
    /// Tool calls begin marker
    pub const TOOL_CALLS_BEGIN: u32 = 128806;
    /// Tool calls end marker
    pub const TOOL_CALLS_END: u32 = 128807;
    /// Single tool call begin
    pub const TOOL_CALL_BEGIN: u32 = 128808;
    /// Single tool call end
    pub const TOOL_CALL_END: u32 = 128809;
    /// Tool outputs begin marker
    pub const TOOL_OUTPUTS_BEGIN: u32 = 128810;
    /// Tool outputs end marker
    pub const TOOL_OUTPUTS_END: u32 = 128811;
    /// Single tool output begin
    pub const TOOL_OUTPUT_BEGIN: u32 = 128812;
    /// Single tool output end
    pub const TOOL_OUTPUT_END: u32 = 128813;
    /// Tool separator
    pub const TOOL_SEP: u32 = 128814;

    // Conversation & Roles
    /// System role - system instructions
    pub const SYSTEM: u32 = 128900;
    /// User role - user input
    pub const USER: u32 = 128901;
    /// Assistant role - model output
    pub const ASSISTANT: u32 = 128902;
    /// Start of message - ChatML wrapper
    pub const IM_START: u32 = 128903;
    /// End of message - ChatML wrapper
    pub const IM_END: u32 = 128904;

    // Reasoning/Thinking
    /// Start of thinking - Chain-of-Thought
    pub const THINK: u32 = 128905;
    /// End of thinking
    pub const THINK_END: u32 = 128906;

    // ReAct Agent Loop
    /// Start of plan - action planning
    pub const PLAN: u32 = 128907;
    /// End of plan
    pub const PLAN_END: u32 = 128908;
    /// Start of step - individual action step
    pub const STEP: u32 = 128909;
    /// End of step
    pub const STEP_END: u32 = 128910;
    /// Start of action - agent action
    pub const ACT: u32 = 128911;
    /// End of action
    pub const ACT_END: u32 = 128912;
    /// Start of observation - environment feedback
    pub const OBSERVE: u32 = 128913;
    /// End of observation
    pub const OBSERVE_END: u32 = 128914;

    // Tool/Function Calling
    /// Start of function call - function invocation
    pub const FUNCTION: u32 = 128915;
    /// End of function call
    pub const FUNCTION_END: u32 = 128916;
    /// Start of function result - return value
    pub const RESULT: u32 = 128917;
    /// End of function result
    pub const RESULT_END: u32 = 128918;
    /// Start of error - error message
    pub const ERROR: u32 = 128919;
    /// End of error
    pub const ERROR_END: u32 = 128920;

    // Code Execution
    /// Start of code - inline code execution
    pub const CODE: u32 = 128921;
    /// End of code
    pub const CODE_END: u32 = 128922;
    /// Start of output - execution output
    pub const OUTPUT: u32 = 128923;
    /// End of output
    pub const OUTPUT_END: u32 = 128924;
    /// Start of language tag - code language
    pub const LANG: u32 = 128925;
    /// End of language tag
    pub const LANG_END: u32 = 128926;

    // RAG & Citations
    /// Start of context - retrieved context
    pub const CONTEXT: u32 = 128927;
    /// End of context
    pub const CONTEXT_END: u32 = 128928;
    /// Start of quote - exact citation
    pub const QUOTE: u32 = 128929;
    /// End of quote
    pub const QUOTE_END: u32 = 128930;
    /// Start of cite - citation reference
    pub const CITE: u32 = 128931;
    /// End of cite
    pub const CITE_END: u32 = 128932;
    /// Start of source - document source
    pub const SOURCE: u32 = 128933;
    /// End of source
    pub const SOURCE_END: u32 = 128934;

    // Memory/State Management
    /// Start of memory - persistent memory
    pub const MEMORY: u32 = 128935;
    /// End of memory
    pub const MEMORY_END: u32 = 128936;
    /// Start of recall - memory retrieval
    pub const RECALL: u32 = 128937;
    /// End of recall
    pub const RECALL_END: u32 = 128938;

    // Control Tokens
    /// Padding token
    pub const PAD: u32 = 128939;
    /// Stop generation token
    pub const STOP: u32 = 128940;
    /// Separator token
    pub const SEP: u32 = 128941;

    // Multimodal Placeholders
    /// Start of image - image placeholder
    pub const IMAGE: u32 = 128942;
    /// End of image
    pub const IMAGE_END: u32 = 128943;
    /// Start of audio - audio placeholder
    pub const AUDIO: u32 = 128944;
    /// End of audio
    pub const AUDIO_END: u32 = 128945;
    /// Start of video - video placeholder
    pub const VIDEO: u32 = 128946;
    /// End of video
    pub const VIDEO_END: u32 = 128947;

    // Document Structure
    /// Start of title - document/section title
    pub const TITLE: u32 = 128948;
    /// End of title
    pub const TITLE_END: u32 = 128949;
    /// Start of section - semantic document section
    pub const SECTION: u32 = 128950;
    /// End of section
    pub const SECTION_END: u32 = 128951;
    /// Start of summary - condensed content summary
    pub const SUMMARY: u32 = 128952;
    /// End of summary
    pub const SUMMARY_END: u32 = 128953;
}

/// Agent tokens for Mistral V1 (7B v0.1/v0.2, Mixtral 8x7B).
///
/// The 54-token block starts at 32000. Every id here is also reachable
/// at runtime as `from_pretrained("mistral_v1")?.special_token_id(name)`;
/// these constants are the compile-time form, so a typo is a compile
/// error rather than a `None` to unwrap.
pub mod mistral_v1_agent_tokens {
    // Conversation & Roles
    /// System role - system instructions
    pub const SYSTEM: u32 = 32000;
    /// User role - user input
    pub const USER: u32 = 32001;
    /// Assistant role - model output
    pub const ASSISTANT: u32 = 32002;
    /// Start of message - ChatML wrapper
    pub const IM_START: u32 = 32003;
    /// End of message - ChatML wrapper
    pub const IM_END: u32 = 32004;

    // Reasoning/Thinking
    /// Start of thinking - Chain-of-Thought
    pub const THINK: u32 = 32005;
    /// End of thinking
    pub const THINK_END: u32 = 32006;

    // ReAct Agent Loop
    /// Start of plan - action planning
    pub const PLAN: u32 = 32007;
    /// End of plan
    pub const PLAN_END: u32 = 32008;
    /// Start of step - individual action step
    pub const STEP: u32 = 32009;
    /// End of step
    pub const STEP_END: u32 = 32010;
    /// Start of action - agent action
    pub const ACT: u32 = 32011;
    /// End of action
    pub const ACT_END: u32 = 32012;
    /// Start of observation - environment feedback
    pub const OBSERVE: u32 = 32013;
    /// End of observation
    pub const OBSERVE_END: u32 = 32014;

    // Tool/Function Calling
    /// Start of function call - function invocation
    pub const FUNCTION: u32 = 32015;
    /// End of function call
    pub const FUNCTION_END: u32 = 32016;
    /// Start of function result - return value
    pub const RESULT: u32 = 32017;
    /// End of function result
    pub const RESULT_END: u32 = 32018;
    /// Start of error - error message
    pub const ERROR: u32 = 32019;
    /// End of error
    pub const ERROR_END: u32 = 32020;

    // Code Execution
    /// Start of code - inline code execution
    pub const CODE: u32 = 32021;
    /// End of code
    pub const CODE_END: u32 = 32022;
    /// Start of output - execution output
    pub const OUTPUT: u32 = 32023;
    /// End of output
    pub const OUTPUT_END: u32 = 32024;
    /// Start of language tag - code language
    pub const LANG: u32 = 32025;
    /// End of language tag
    pub const LANG_END: u32 = 32026;

    // RAG & Citations
    /// Start of context - retrieved context
    pub const CONTEXT: u32 = 32027;
    /// End of context
    pub const CONTEXT_END: u32 = 32028;
    /// Start of quote - exact citation
    pub const QUOTE: u32 = 32029;
    /// End of quote
    pub const QUOTE_END: u32 = 32030;
    /// Start of cite - citation reference
    pub const CITE: u32 = 32031;
    /// End of cite
    pub const CITE_END: u32 = 32032;
    /// Start of source - document source
    pub const SOURCE: u32 = 32033;
    /// End of source
    pub const SOURCE_END: u32 = 32034;

    // Memory/State Management
    /// Start of memory - persistent memory
    pub const MEMORY: u32 = 32035;
    /// End of memory
    pub const MEMORY_END: u32 = 32036;
    /// Start of recall - memory retrieval
    pub const RECALL: u32 = 32037;
    /// End of recall
    pub const RECALL_END: u32 = 32038;

    // Control Tokens
    /// Padding token
    pub const PAD: u32 = 32039;
    /// Stop generation token
    pub const STOP: u32 = 32040;
    /// Separator token
    pub const SEP: u32 = 32041;

    // Multimodal Placeholders
    /// Start of image - image placeholder
    pub const IMAGE: u32 = 32042;
    /// End of image
    pub const IMAGE_END: u32 = 32043;
    /// Start of audio - audio placeholder
    pub const AUDIO: u32 = 32044;
    /// End of audio
    pub const AUDIO_END: u32 = 32045;
    /// Start of video - video placeholder
    pub const VIDEO: u32 = 32046;
    /// End of video
    pub const VIDEO_END: u32 = 32047;

    // Document Structure
    /// Start of title - document/section title
    pub const TITLE: u32 = 32048;
    /// End of title
    pub const TITLE_END: u32 = 32049;
    /// Start of section - semantic document section
    pub const SECTION: u32 = 32050;
    /// End of section
    pub const SECTION_END: u32 = 32051;
    /// Start of summary - condensed content summary
    pub const SUMMARY: u32 = 32052;
    /// End of summary
    pub const SUMMARY_END: u32 = 32053;
}

/// Agent tokens for Mistral V2 (7B v0.3, Mixtral 8x22B, Codestral).
///
/// The 54-token block starts at 32768. Every id here is also reachable
/// at runtime as `from_pretrained("mistral_v2")?.special_token_id(name)`;
/// these constants are the compile-time form, so a typo is a compile
/// error rather than a `None` to unwrap.
pub mod mistral_v2_agent_tokens {
    // Conversation & Roles
    /// System role - system instructions
    pub const SYSTEM: u32 = 32768;
    /// User role - user input
    pub const USER: u32 = 32769;
    /// Assistant role - model output
    pub const ASSISTANT: u32 = 32770;
    /// Start of message - ChatML wrapper
    pub const IM_START: u32 = 32771;
    /// End of message - ChatML wrapper
    pub const IM_END: u32 = 32772;

    // Reasoning/Thinking
    /// Start of thinking - Chain-of-Thought
    pub const THINK: u32 = 32773;
    /// End of thinking
    pub const THINK_END: u32 = 32774;

    // ReAct Agent Loop
    /// Start of plan - action planning
    pub const PLAN: u32 = 32775;
    /// End of plan
    pub const PLAN_END: u32 = 32776;
    /// Start of step - individual action step
    pub const STEP: u32 = 32777;
    /// End of step
    pub const STEP_END: u32 = 32778;
    /// Start of action - agent action
    pub const ACT: u32 = 32779;
    /// End of action
    pub const ACT_END: u32 = 32780;
    /// Start of observation - environment feedback
    pub const OBSERVE: u32 = 32781;
    /// End of observation
    pub const OBSERVE_END: u32 = 32782;

    // Tool/Function Calling
    /// Start of function call - function invocation
    pub const FUNCTION: u32 = 32783;
    /// End of function call
    pub const FUNCTION_END: u32 = 32784;
    /// Start of function result - return value
    pub const RESULT: u32 = 32785;
    /// End of function result
    pub const RESULT_END: u32 = 32786;
    /// Start of error - error message
    pub const ERROR: u32 = 32787;
    /// End of error
    pub const ERROR_END: u32 = 32788;

    // Code Execution
    /// Start of code - inline code execution
    pub const CODE: u32 = 32789;
    /// End of code
    pub const CODE_END: u32 = 32790;
    /// Start of output - execution output
    pub const OUTPUT: u32 = 32791;
    /// End of output
    pub const OUTPUT_END: u32 = 32792;
    /// Start of language tag - code language
    pub const LANG: u32 = 32793;
    /// End of language tag
    pub const LANG_END: u32 = 32794;

    // RAG & Citations
    /// Start of context - retrieved context
    pub const CONTEXT: u32 = 32795;
    /// End of context
    pub const CONTEXT_END: u32 = 32796;
    /// Start of quote - exact citation
    pub const QUOTE: u32 = 32797;
    /// End of quote
    pub const QUOTE_END: u32 = 32798;
    /// Start of cite - citation reference
    pub const CITE: u32 = 32799;
    /// End of cite
    pub const CITE_END: u32 = 32800;
    /// Start of source - document source
    pub const SOURCE: u32 = 32801;
    /// End of source
    pub const SOURCE_END: u32 = 32802;

    // Memory/State Management
    /// Start of memory - persistent memory
    pub const MEMORY: u32 = 32803;
    /// End of memory
    pub const MEMORY_END: u32 = 32804;
    /// Start of recall - memory retrieval
    pub const RECALL: u32 = 32805;
    /// End of recall
    pub const RECALL_END: u32 = 32806;

    // Control Tokens
    /// Padding token
    pub const PAD: u32 = 32807;
    /// Stop generation token
    pub const STOP: u32 = 32808;
    /// Separator token
    pub const SEP: u32 = 32809;

    // Multimodal Placeholders
    /// Start of image - image placeholder
    pub const IMAGE: u32 = 32810;
    /// End of image
    pub const IMAGE_END: u32 = 32811;
    /// Start of audio - audio placeholder
    pub const AUDIO: u32 = 32812;
    /// End of audio
    pub const AUDIO_END: u32 = 32813;
    /// Start of video - video placeholder
    pub const VIDEO: u32 = 32814;
    /// End of video
    pub const VIDEO_END: u32 = 32815;

    // Document Structure
    /// Start of title - document/section title
    pub const TITLE: u32 = 32816;
    /// End of title
    pub const TITLE_END: u32 = 32817;
    /// Start of section - semantic document section
    pub const SECTION: u32 = 32818;
    /// End of section
    pub const SECTION_END: u32 = 32819;
    /// Start of summary - condensed content summary
    pub const SUMMARY: u32 = 32820;
    /// End of summary
    pub const SUMMARY_END: u32 = 32821;
}

/// Agent tokens for Mistral V3/Tekken (NeMo, Large 2, Pixtral).
///
/// The 54-token block starts at 131072. Every id here is also reachable
/// at runtime as `from_pretrained("mistral_v3")?.special_token_id(name)`;
/// these constants are the compile-time form, so a typo is a compile
/// error rather than a `None` to unwrap.
///
/// The vocabulary's own markers are included too, so this module is its
/// whole special-token surface. A native name that would collide with
/// an agent token of the same meaning takes a `_NATIVE` suffix.
pub mod mistral_v3_agent_tokens {
    // Vocabulary-native tokens
    /// `[INST]` instruction begin
    pub const INST: u32 = 3;
    /// `[/INST]` instruction end
    pub const INST_END: u32 = 4;
    /// `[AVAILABLE_TOOLS]` tool list begin
    pub const AVAILABLE_TOOLS: u32 = 5;
    /// `[/AVAILABLE_TOOLS]` tool list end
    pub const AVAILABLE_TOOLS_END: u32 = 6;
    /// `[TOOL_RESULTS]` tool results begin
    pub const TOOL_RESULTS: u32 = 7;
    /// `[/TOOL_RESULTS]` tool results end
    pub const TOOL_RESULTS_END: u32 = 8;
    /// `[TOOL_CALLS]` tool calls marker
    pub const TOOL_CALLS: u32 = 9;

    // Conversation & Roles
    /// System role - system instructions
    pub const SYSTEM: u32 = 131072;
    /// User role - user input
    pub const USER: u32 = 131073;
    /// Assistant role - model output
    pub const ASSISTANT: u32 = 131074;
    /// Start of message - ChatML wrapper
    pub const IM_START: u32 = 131075;
    /// End of message - ChatML wrapper
    pub const IM_END: u32 = 131076;

    // Reasoning/Thinking
    /// Start of thinking - Chain-of-Thought
    pub const THINK: u32 = 131077;
    /// End of thinking
    pub const THINK_END: u32 = 131078;

    // ReAct Agent Loop
    /// Start of plan - action planning
    pub const PLAN: u32 = 131079;
    /// End of plan
    pub const PLAN_END: u32 = 131080;
    /// Start of step - individual action step
    pub const STEP: u32 = 131081;
    /// End of step
    pub const STEP_END: u32 = 131082;
    /// Start of action - agent action
    pub const ACT: u32 = 131083;
    /// End of action
    pub const ACT_END: u32 = 131084;
    /// Start of observation - environment feedback
    pub const OBSERVE: u32 = 131085;
    /// End of observation
    pub const OBSERVE_END: u32 = 131086;

    // Tool/Function Calling
    /// Start of function call - function invocation
    pub const FUNCTION: u32 = 131087;
    /// End of function call
    pub const FUNCTION_END: u32 = 131088;
    /// Start of function result - return value
    pub const RESULT: u32 = 131089;
    /// End of function result
    pub const RESULT_END: u32 = 131090;
    /// Start of error - error message
    pub const ERROR: u32 = 131091;
    /// End of error
    pub const ERROR_END: u32 = 131092;

    // Code Execution
    /// Start of code - inline code execution
    pub const CODE: u32 = 131093;
    /// End of code
    pub const CODE_END: u32 = 131094;
    /// Start of output - execution output
    pub const OUTPUT: u32 = 131095;
    /// End of output
    pub const OUTPUT_END: u32 = 131096;
    /// Start of language tag - code language
    pub const LANG: u32 = 131097;
    /// End of language tag
    pub const LANG_END: u32 = 131098;

    // RAG & Citations
    /// Start of context - retrieved context
    pub const CONTEXT: u32 = 131099;
    /// End of context
    pub const CONTEXT_END: u32 = 131100;
    /// Start of quote - exact citation
    pub const QUOTE: u32 = 131101;
    /// End of quote
    pub const QUOTE_END: u32 = 131102;
    /// Start of cite - citation reference
    pub const CITE: u32 = 131103;
    /// End of cite
    pub const CITE_END: u32 = 131104;
    /// Start of source - document source
    pub const SOURCE: u32 = 131105;
    /// End of source
    pub const SOURCE_END: u32 = 131106;

    // Memory/State Management
    /// Start of memory - persistent memory
    pub const MEMORY: u32 = 131107;
    /// End of memory
    pub const MEMORY_END: u32 = 131108;
    /// Start of recall - memory retrieval
    pub const RECALL: u32 = 131109;
    /// End of recall
    pub const RECALL_END: u32 = 131110;

    // Control Tokens
    /// Padding token
    pub const PAD: u32 = 131111;
    /// Stop generation token
    pub const STOP: u32 = 131112;
    /// Separator token
    pub const SEP: u32 = 131113;

    // Multimodal Placeholders
    /// Start of image - image placeholder
    pub const IMAGE: u32 = 131114;
    /// End of image
    pub const IMAGE_END: u32 = 131115;
    /// Start of audio - audio placeholder
    pub const AUDIO: u32 = 131116;
    /// End of audio
    pub const AUDIO_END: u32 = 131117;
    /// Start of video - video placeholder
    pub const VIDEO: u32 = 131118;
    /// End of video
    pub const VIDEO_END: u32 = 131119;

    // Document Structure
    /// Start of title - document/section title
    pub const TITLE: u32 = 131120;
    /// End of title
    pub const TITLE_END: u32 = 131121;
    /// Start of section - semantic document section
    pub const SECTION: u32 = 131122;
    /// End of section
    pub const SECTION_END: u32 = 131123;
    /// Start of summary - condensed content summary
    pub const SUMMARY: u32 = 131124;
    /// End of summary
    pub const SUMMARY_END: u32 = 131125;
}

/// Agent tokens for Qwen 2/3 (also Baichuan-M2).
///
/// The 54-token block starts at 151669. Every id here is also reachable
/// at runtime as `from_pretrained("qwen3")?.special_token_id(name)`;
/// these constants are the compile-time form, so a typo is a compile
/// error rather than a `None` to unwrap.
///
/// Qwen 2/3 (also Baichuan-M2) defines IM_END, IM_START itself, so those constants carry the
/// vocabulary's own id — below `base_vocab_size`, the id the checkpoint
/// was trained on — rather than a splintr-appended one.
///
/// The vocabulary's own markers are included too, so this module is its
/// whole special-token surface. A native name that would collide with
/// an agent token of the same meaning takes a `_NATIVE` suffix.
pub mod qwen3_agent_tokens {
    // Vocabulary-native tokens
    /// End of text marker
    pub const ENDOFTEXT: u32 = 151643;
    /// Native `<|im_start|>` - ChatML message start
    pub const IM_START: u32 = 151644;
    /// Native `<|im_end|>` - ChatML message end
    pub const IM_END: u32 = 151645;
    /// Object reference begin
    pub const OBJECT_REF_START: u32 = 151646;
    /// Object reference end
    pub const OBJECT_REF_END: u32 = 151647;
    /// Bounding box begin
    pub const BOX_START: u32 = 151648;
    /// Bounding box end
    pub const BOX_END: u32 = 151649;
    /// Quadrilateral begin
    pub const QUAD_START: u32 = 151650;
    /// Quadrilateral end
    pub const QUAD_END: u32 = 151651;
    /// Vision block begin
    pub const VISION_START: u32 = 151652;
    /// Vision block end
    pub const VISION_END: u32 = 151653;
    /// Vision padding
    pub const VISION_PAD: u32 = 151654;
    /// Image padding
    pub const IMAGE_PAD: u32 = 151655;
    /// Video padding
    pub const VIDEO_PAD: u32 = 151656;
    /// Native `<tool_call>`
    pub const TOOL_CALL: u32 = 151657;
    /// Native `</tool_call>`
    pub const TOOL_CALL_END: u32 = 151658;
    /// Fill-in-the-Middle prefix
    pub const FIM_PREFIX: u32 = 151659;
    /// Fill-in-the-Middle middle
    pub const FIM_MIDDLE: u32 = 151660;
    /// Fill-in-the-Middle suffix
    pub const FIM_SUFFIX: u32 = 151661;
    /// Fill-in-the-Middle padding
    pub const FIM_PAD: u32 = 151662;
    /// Repository name marker
    pub const REPO_NAME: u32 = 151663;
    /// File separator
    pub const FILE_SEP: u32 = 151664;
    /// Native `<tool_response>`
    pub const TOOL_RESPONSE: u32 = 151665;
    /// Native `</tool_response>`
    pub const TOOL_RESPONSE_END: u32 = 151666;
    /// Native `<think>` token
    pub const THINK_NATIVE: u32 = 151667;
    /// Native `</think>` token
    pub const THINK_END_NATIVE: u32 = 151668;

    // Conversation & Roles
    /// System role - system instructions
    pub const SYSTEM: u32 = 151669;
    /// User role - user input
    pub const USER: u32 = 151670;
    /// Assistant role - model output
    pub const ASSISTANT: u32 = 151671;

    // Reasoning/Thinking
    /// Start of thinking - Chain-of-Thought
    pub const THINK: u32 = 151674;
    /// End of thinking
    pub const THINK_END: u32 = 151675;

    // ReAct Agent Loop
    /// Start of plan - action planning
    pub const PLAN: u32 = 151676;
    /// End of plan
    pub const PLAN_END: u32 = 151677;
    /// Start of step - individual action step
    pub const STEP: u32 = 151678;
    /// End of step
    pub const STEP_END: u32 = 151679;
    /// Start of action - agent action
    pub const ACT: u32 = 151680;
    /// End of action
    pub const ACT_END: u32 = 151681;
    /// Start of observation - environment feedback
    pub const OBSERVE: u32 = 151682;
    /// End of observation
    pub const OBSERVE_END: u32 = 151683;

    // Tool/Function Calling
    /// Start of function call - function invocation
    pub const FUNCTION: u32 = 151684;
    /// End of function call
    pub const FUNCTION_END: u32 = 151685;
    /// Start of function result - return value
    pub const RESULT: u32 = 151686;
    /// End of function result
    pub const RESULT_END: u32 = 151687;
    /// Start of error - error message
    pub const ERROR: u32 = 151688;
    /// End of error
    pub const ERROR_END: u32 = 151689;

    // Code Execution
    /// Start of code - inline code execution
    pub const CODE: u32 = 151690;
    /// End of code
    pub const CODE_END: u32 = 151691;
    /// Start of output - execution output
    pub const OUTPUT: u32 = 151692;
    /// End of output
    pub const OUTPUT_END: u32 = 151693;
    /// Start of language tag - code language
    pub const LANG: u32 = 151694;
    /// End of language tag
    pub const LANG_END: u32 = 151695;

    // RAG & Citations
    /// Start of context - retrieved context
    pub const CONTEXT: u32 = 151696;
    /// End of context
    pub const CONTEXT_END: u32 = 151697;
    /// Start of quote - exact citation
    pub const QUOTE: u32 = 151698;
    /// End of quote
    pub const QUOTE_END: u32 = 151699;
    /// Start of cite - citation reference
    pub const CITE: u32 = 151700;
    /// End of cite
    pub const CITE_END: u32 = 151701;
    /// Start of source - document source
    pub const SOURCE: u32 = 151702;
    /// End of source
    pub const SOURCE_END: u32 = 151703;

    // Memory/State Management
    /// Start of memory - persistent memory
    pub const MEMORY: u32 = 151704;
    /// End of memory
    pub const MEMORY_END: u32 = 151705;
    /// Start of recall - memory retrieval
    pub const RECALL: u32 = 151706;
    /// End of recall
    pub const RECALL_END: u32 = 151707;

    // Control Tokens
    /// Padding token
    pub const PAD: u32 = 151708;
    /// Stop generation token
    pub const STOP: u32 = 151709;
    /// Separator token
    pub const SEP: u32 = 151710;

    // Multimodal Placeholders
    /// Start of image - image placeholder
    pub const IMAGE: u32 = 151711;
    /// End of image
    pub const IMAGE_END: u32 = 151712;
    /// Start of audio - audio placeholder
    pub const AUDIO: u32 = 151713;
    /// End of audio
    pub const AUDIO_END: u32 = 151714;
    /// Start of video - video placeholder
    pub const VIDEO: u32 = 151715;
    /// End of video
    pub const VIDEO_END: u32 = 151716;

    // Document Structure
    /// Start of title - document/section title
    pub const TITLE: u32 = 151717;
    /// End of title
    pub const TITLE_END: u32 = 151718;
    /// Start of section - semantic document section
    pub const SECTION: u32 = 151719;
    /// End of section
    pub const SECTION_END: u32 = 151720;
    /// Start of summary - condensed content summary
    pub const SUMMARY: u32 = 151721;
    /// End of summary
    pub const SUMMARY_END: u32 = 151722;
}

/// Agent tokens for GLM-4/4.5.
///
/// The 54-token block starts at 151365. Every id here is also reachable
/// at runtime as `from_pretrained("glm4")?.special_token_id(name)`;
/// these constants are the compile-time form, so a typo is a compile
/// error rather than a `None` to unwrap.
///
/// GLM-4/4.5 defines ASSISTANT, IMAGE, SYSTEM, USER, VIDEO itself, so those constants carry the
/// vocabulary's own id — below `base_vocab_size`, the id the checkpoint
/// was trained on — rather than a splintr-appended one.
///
/// The vocabulary's own markers are included too, so this module is its
/// whole special-token surface. A native name that would collide with
/// an agent token of the same meaning takes a `_NATIVE` suffix.
pub mod glm4_agent_tokens {
    // Vocabulary-native tokens
    /// End of text marker
    pub const ENDOFTEXT: u32 = 151329;
    /// `[MASK]`
    pub const MASK: u32 = 151330;
    /// `[gMASK]`
    pub const GMASK: u32 = 151331;
    /// `[sMASK]`
    pub const SMASK: u32 = 151332;
    /// Start of prefix `<sop>`
    pub const SOP: u32 = 151333;
    /// End of prefix `<eop>`
    pub const EOP: u32 = 151334;
    /// Native `<|system|>` role marker
    pub const SYSTEM: u32 = 151335;
    /// Native `<|user|>` role marker
    pub const USER: u32 = 151336;
    /// Native `<|assistant|>` role marker
    pub const ASSISTANT: u32 = 151337;
    /// Observation role marker
    pub const OBSERVATION: u32 = 151338;
    /// Image block begin
    pub const BEGIN_OF_IMAGE: u32 = 151339;
    /// Image block end
    pub const END_OF_IMAGE: u32 = 151340;
    /// Video block begin
    pub const BEGIN_OF_VIDEO: u32 = 151341;
    /// Video block end
    pub const END_OF_VIDEO: u32 = 151342;
    /// Audio block begin
    pub const BEGIN_OF_AUDIO: u32 = 151343;
    /// Audio block end
    pub const END_OF_AUDIO: u32 = 151344;
    /// Transcription begin
    pub const BEGIN_OF_TRANSCRIPTION: u32 = 151345;
    /// Transcription end
    pub const END_OF_TRANSCRIPTION: u32 = 151346;
    /// Code Fill-in-the-Middle prefix
    pub const CODE_PREFIX: u32 = 151347;
    /// Code Fill-in-the-Middle middle
    pub const CODE_MIDDLE: u32 = 151348;
    /// Code Fill-in-the-Middle suffix
    pub const CODE_SUFFIX: u32 = 151349;
    /// Native `<think>` token
    pub const THINK_NATIVE: u32 = 151350;
    /// Native `</think>` token
    pub const THINK_END_NATIVE: u32 = 151351;
    /// Native `<tool_call>`
    pub const TOOL_CALL: u32 = 151352;
    /// Native `</tool_call>`
    pub const TOOL_CALL_END: u32 = 151353;
    /// Native `<tool_response>`
    pub const TOOL_RESPONSE: u32 = 151354;
    /// Native `</tool_response>`
    pub const TOOL_RESPONSE_END: u32 = 151355;
    /// Tool argument key begin
    pub const ARG_KEY: u32 = 151356;
    /// Tool argument key end
    pub const ARG_KEY_END: u32 = 151357;
    /// Tool argument value begin
    pub const ARG_VALUE: u32 = 151358;
    /// Tool argument value end
    pub const ARG_VALUE_END: u32 = 151359;
    /// /nothink directive
    pub const NOTHINK: u32 = 151360;
    /// Box begin
    pub const BEGIN_OF_BOX: u32 = 151361;
    /// Box end
    pub const END_OF_BOX: u32 = 151362;
    /// Native `<|image|>` placeholder
    pub const IMAGE: u32 = 151363;
    /// Native `<|video|>` placeholder
    pub const VIDEO: u32 = 151364;

    // Conversation & Roles
    /// Start of message - ChatML wrapper
    pub const IM_START: u32 = 151368;
    /// End of message - ChatML wrapper
    pub const IM_END: u32 = 151369;

    // Reasoning/Thinking
    /// Start of thinking - Chain-of-Thought
    pub const THINK: u32 = 151370;
    /// End of thinking
    pub const THINK_END: u32 = 151371;

    // ReAct Agent Loop
    /// Start of plan - action planning
    pub const PLAN: u32 = 151372;
    /// End of plan
    pub const PLAN_END: u32 = 151373;
    /// Start of step - individual action step
    pub const STEP: u32 = 151374;
    /// End of step
    pub const STEP_END: u32 = 151375;
    /// Start of action - agent action
    pub const ACT: u32 = 151376;
    /// End of action
    pub const ACT_END: u32 = 151377;
    /// Start of observation - environment feedback
    pub const OBSERVE: u32 = 151378;
    /// End of observation
    pub const OBSERVE_END: u32 = 151379;

    // Tool/Function Calling
    /// Start of function call - function invocation
    pub const FUNCTION: u32 = 151380;
    /// End of function call
    pub const FUNCTION_END: u32 = 151381;
    /// Start of function result - return value
    pub const RESULT: u32 = 151382;
    /// End of function result
    pub const RESULT_END: u32 = 151383;
    /// Start of error - error message
    pub const ERROR: u32 = 151384;
    /// End of error
    pub const ERROR_END: u32 = 151385;

    // Code Execution
    /// Start of code - inline code execution
    pub const CODE: u32 = 151386;
    /// End of code
    pub const CODE_END: u32 = 151387;
    /// Start of output - execution output
    pub const OUTPUT: u32 = 151388;
    /// End of output
    pub const OUTPUT_END: u32 = 151389;
    /// Start of language tag - code language
    pub const LANG: u32 = 151390;
    /// End of language tag
    pub const LANG_END: u32 = 151391;

    // RAG & Citations
    /// Start of context - retrieved context
    pub const CONTEXT: u32 = 151392;
    /// End of context
    pub const CONTEXT_END: u32 = 151393;
    /// Start of quote - exact citation
    pub const QUOTE: u32 = 151394;
    /// End of quote
    pub const QUOTE_END: u32 = 151395;
    /// Start of cite - citation reference
    pub const CITE: u32 = 151396;
    /// End of cite
    pub const CITE_END: u32 = 151397;
    /// Start of source - document source
    pub const SOURCE: u32 = 151398;
    /// End of source
    pub const SOURCE_END: u32 = 151399;

    // Memory/State Management
    /// Start of memory - persistent memory
    pub const MEMORY: u32 = 151400;
    /// End of memory
    pub const MEMORY_END: u32 = 151401;
    /// Start of recall - memory retrieval
    pub const RECALL: u32 = 151402;
    /// End of recall
    pub const RECALL_END: u32 = 151403;

    // Control Tokens
    /// Padding token
    pub const PAD: u32 = 151404;
    /// Stop generation token
    pub const STOP: u32 = 151405;
    /// Separator token
    pub const SEP: u32 = 151406;

    // Multimodal Placeholders
    /// End of image
    pub const IMAGE_END: u32 = 151408;
    /// Start of audio - audio placeholder
    pub const AUDIO: u32 = 151409;
    /// End of audio
    pub const AUDIO_END: u32 = 151410;
    /// End of video
    pub const VIDEO_END: u32 = 151412;

    // Document Structure
    /// Start of title - document/section title
    pub const TITLE: u32 = 151413;
    /// End of title
    pub const TITLE_END: u32 = 151414;
    /// Start of section - semantic document section
    pub const SECTION: u32 = 151415;
    /// End of section
    pub const SECTION_END: u32 = 151416;
    /// Start of summary - condensed content summary
    pub const SUMMARY: u32 = 151417;
    /// End of summary
    pub const SUMMARY_END: u32 = 151418;
}

/// Agent tokens for OpenAI gpt-oss.
///
/// The 54-token block starts at 200019. Every id here is also reachable
/// at runtime as `from_pretrained("gpt-oss")?.special_token_id(name)`;
/// these constants are the compile-time form, so a typo is a compile
/// error rather than a `None` to unwrap.
///
/// The vocabulary's own markers are included too, so this module is its
/// whole special-token surface. A native name that would collide with
/// an agent token of the same meaning takes a `_NATIVE` suffix.
pub mod gpt_oss_agent_tokens {
    // Vocabulary-native tokens
    /// Start of text marker
    pub const STARTOFTEXT: u32 = 199998;
    /// End of text marker
    pub const ENDOFTEXT: u32 = 199999;
    /// End of a final assistant turn
    pub const RETURN: u32 = 200002;
    /// Constrained-output marker
    pub const CONSTRAIN: u32 = 200003;
    /// Channel marker (analysis/commentary/final)
    pub const CHANNEL: u32 = 200005;
    /// Message start
    pub const START: u32 = 200006;
    /// Message end
    pub const END: u32 = 200007;
    /// Message body begin
    pub const MESSAGE: u32 = 200008;
    /// Tool call marker
    pub const CALL: u32 = 200012;
    /// End of prompt
    pub const ENDOFPROMPT: u32 = 200018;

    // Conversation & Roles
    /// System role - system instructions
    pub const SYSTEM: u32 = 200019;
    /// User role - user input
    pub const USER: u32 = 200020;
    /// Assistant role - model output
    pub const ASSISTANT: u32 = 200021;
    /// Start of message - ChatML wrapper
    pub const IM_START: u32 = 200022;
    /// End of message - ChatML wrapper
    pub const IM_END: u32 = 200023;

    // Reasoning/Thinking
    /// Start of thinking - Chain-of-Thought
    pub const THINK: u32 = 200024;
    /// End of thinking
    pub const THINK_END: u32 = 200025;

    // ReAct Agent Loop
    /// Start of plan - action planning
    pub const PLAN: u32 = 200026;
    /// End of plan
    pub const PLAN_END: u32 = 200027;
    /// Start of step - individual action step
    pub const STEP: u32 = 200028;
    /// End of step
    pub const STEP_END: u32 = 200029;
    /// Start of action - agent action
    pub const ACT: u32 = 200030;
    /// End of action
    pub const ACT_END: u32 = 200031;
    /// Start of observation - environment feedback
    pub const OBSERVE: u32 = 200032;
    /// End of observation
    pub const OBSERVE_END: u32 = 200033;

    // Tool/Function Calling
    /// Start of function call - function invocation
    pub const FUNCTION: u32 = 200034;
    /// End of function call
    pub const FUNCTION_END: u32 = 200035;
    /// Start of function result - return value
    pub const RESULT: u32 = 200036;
    /// End of function result
    pub const RESULT_END: u32 = 200037;
    /// Start of error - error message
    pub const ERROR: u32 = 200038;
    /// End of error
    pub const ERROR_END: u32 = 200039;

    // Code Execution
    /// Start of code - inline code execution
    pub const CODE: u32 = 200040;
    /// End of code
    pub const CODE_END: u32 = 200041;
    /// Start of output - execution output
    pub const OUTPUT: u32 = 200042;
    /// End of output
    pub const OUTPUT_END: u32 = 200043;
    /// Start of language tag - code language
    pub const LANG: u32 = 200044;
    /// End of language tag
    pub const LANG_END: u32 = 200045;

    // RAG & Citations
    /// Start of context - retrieved context
    pub const CONTEXT: u32 = 200046;
    /// End of context
    pub const CONTEXT_END: u32 = 200047;
    /// Start of quote - exact citation
    pub const QUOTE: u32 = 200048;
    /// End of quote
    pub const QUOTE_END: u32 = 200049;
    /// Start of cite - citation reference
    pub const CITE: u32 = 200050;
    /// End of cite
    pub const CITE_END: u32 = 200051;
    /// Start of source - document source
    pub const SOURCE: u32 = 200052;
    /// End of source
    pub const SOURCE_END: u32 = 200053;

    // Memory/State Management
    /// Start of memory - persistent memory
    pub const MEMORY: u32 = 200054;
    /// End of memory
    pub const MEMORY_END: u32 = 200055;
    /// Start of recall - memory retrieval
    pub const RECALL: u32 = 200056;
    /// End of recall
    pub const RECALL_END: u32 = 200057;

    // Control Tokens
    /// Padding token
    pub const PAD: u32 = 200058;
    /// Stop generation token
    pub const STOP: u32 = 200059;
    /// Separator token
    pub const SEP: u32 = 200060;

    // Multimodal Placeholders
    /// Start of image - image placeholder
    pub const IMAGE: u32 = 200061;
    /// End of image
    pub const IMAGE_END: u32 = 200062;
    /// Start of audio - audio placeholder
    pub const AUDIO: u32 = 200063;
    /// End of audio
    pub const AUDIO_END: u32 = 200064;
    /// Start of video - video placeholder
    pub const VIDEO: u32 = 200065;
    /// End of video
    pub const VIDEO_END: u32 = 200066;

    // Document Structure
    /// Start of title - document/section title
    pub const TITLE: u32 = 200067;
    /// End of title
    pub const TITLE_END: u32 = 200068;
    /// Start of section - semantic document section
    pub const SECTION: u32 = 200069;
    /// End of section
    pub const SECTION_END: u32 = 200070;
    /// Start of summary - condensed content summary
    pub const SUMMARY: u32 = 200071;
    /// End of summary
    pub const SUMMARY_END: u32 = 200072;
}