splintr 0.9.1

// =============================================================================
// AUTO-GENERATED FILE - DO NOT EDIT MANUALLY
// Generated by: scripts/generate_agent_tokens.py
// Source of truth: AGENT_TOKENS in generate_agent_tokens.py
// =============================================================================

// Note: pyo3::prelude::* is already imported in bindings.rs

/// cl100k_base (GPT-4, GPT-3.5-turbo) Agent Token IDs (100277-100330)
///
/// Access agent token IDs for cl100k_base (GPT-4, GPT-3.5-turbo).
///
/// # Examples
///
/// ```python
/// from splintr import Tokenizer, CL100K_AGENT_TOKENS
///
/// tokenizer = Tokenizer.from_pretrained("cl100k_base")
/// system_id = CL100K_AGENT_TOKENS.SYSTEM  # 100277
/// think_id = CL100K_AGENT_TOKENS.THINK   # 100282
///
/// text = "<|system|>You are a helpful assistant"
/// tokens = tokenizer.encode_with_special(text)
/// assert CL100K_AGENT_TOKENS.SYSTEM in tokens
/// ```
#[pyclass(name = "CL100K_AGENT_TOKENS", frozen)]
pub struct PyCL100KAgentTokens;

#[pymethods]
impl PyCL100KAgentTokens {
    // =========================================================================
    // Conversation & Roles (100277-100281)
    // =========================================================================

    /// System role - system instructions (100277)
    #[classattr]
    const SYSTEM: u32 = 100277;
    /// User role - user input (100278)
    #[classattr]
    const USER: u32 = 100278;
    /// Assistant role - model output (100279)
    #[classattr]
    const ASSISTANT: u32 = 100279;
    /// Start of message - ChatML wrapper (100280)
    #[classattr]
    const IM_START: u32 = 100280;
    /// End of message - ChatML wrapper (100281)
    #[classattr]
    const IM_END: u32 = 100281;

    // =========================================================================
    // Reasoning/Thinking (100282-100283)
    // =========================================================================

    /// Start of thinking - Chain-of-Thought (100282)
    #[classattr]
    const THINK: u32 = 100282;
    /// End of thinking (100283)
    #[classattr]
    const THINK_END: u32 = 100283;

    // =========================================================================
    // ReAct Agent Loop (100284-100291)
    // =========================================================================

    /// Start of plan - action planning (100284)
    #[classattr]
    const PLAN: u32 = 100284;
    /// End of plan (100285)
    #[classattr]
    const PLAN_END: u32 = 100285;
    /// Start of step - individual action step (100286)
    #[classattr]
    const STEP: u32 = 100286;
    /// End of step (100287)
    #[classattr]
    const STEP_END: u32 = 100287;
    /// Start of action - agent action (100288)
    #[classattr]
    const ACT: u32 = 100288;
    /// End of action (100289)
    #[classattr]
    const ACT_END: u32 = 100289;
    /// Start of observation - environment feedback (100290)
    #[classattr]
    const OBSERVE: u32 = 100290;
    /// End of observation (100291)
    #[classattr]
    const OBSERVE_END: u32 = 100291;

    // =========================================================================
    // Tool/Function Calling (100292-100297)
    // =========================================================================

    /// Start of function call - function invocation (100292)
    #[classattr]
    const FUNCTION: u32 = 100292;
    /// End of function call (100293)
    #[classattr]
    const FUNCTION_END: u32 = 100293;
    /// Start of function result - return value (100294)
    #[classattr]
    const RESULT: u32 = 100294;
    /// End of function result (100295)
    #[classattr]
    const RESULT_END: u32 = 100295;
    /// Start of error - error message (100296)
    #[classattr]
    const ERROR: u32 = 100296;
    /// End of error (100297)
    #[classattr]
    const ERROR_END: u32 = 100297;

    // =========================================================================
    // Code Execution (100298-100303)
    // =========================================================================

    /// Start of code - inline code execution (100298)
    #[classattr]
    const CODE: u32 = 100298;
    /// End of code (100299)
    #[classattr]
    const CODE_END: u32 = 100299;
    /// Start of output - execution output (100300)
    #[classattr]
    const OUTPUT: u32 = 100300;
    /// End of output (100301)
    #[classattr]
    const OUTPUT_END: u32 = 100301;
    /// Start of language tag - code language (100302)
    #[classattr]
    const LANG: u32 = 100302;
    /// End of language tag (100303)
    #[classattr]
    const LANG_END: u32 = 100303;

    // =========================================================================
    // RAG & Citations (100304-100311)
    // =========================================================================

    /// Start of context - retrieved context (100304)
    #[classattr]
    const CONTEXT: u32 = 100304;
    /// End of context (100305)
    #[classattr]
    const CONTEXT_END: u32 = 100305;
    /// Start of quote - exact citation (100306)
    #[classattr]
    const QUOTE: u32 = 100306;
    /// End of quote (100307)
    #[classattr]
    const QUOTE_END: u32 = 100307;
    /// Start of cite - citation reference (100308)
    #[classattr]
    const CITE: u32 = 100308;
    /// End of cite (100309)
    #[classattr]
    const CITE_END: u32 = 100309;
    /// Start of source - document source (100310)
    #[classattr]
    const SOURCE: u32 = 100310;
    /// End of source (100311)
    #[classattr]
    const SOURCE_END: u32 = 100311;

    // =========================================================================
    // Memory/State Management (100312-100315)
    // =========================================================================

    /// Start of memory - persistent memory (100312)
    #[classattr]
    const MEMORY: u32 = 100312;
    /// End of memory (100313)
    #[classattr]
    const MEMORY_END: u32 = 100313;
    /// Start of recall - memory retrieval (100314)
    #[classattr]
    const RECALL: u32 = 100314;
    /// End of recall (100315)
    #[classattr]
    const RECALL_END: u32 = 100315;

    // =========================================================================
    // Control Tokens (100316-100318)
    // =========================================================================

    /// Padding token (100316)
    #[classattr]
    const PAD: u32 = 100316;
    /// Stop generation token (100317)
    #[classattr]
    const STOP: u32 = 100317;
    /// Separator token (100318)
    #[classattr]
    const SEP: u32 = 100318;

    // =========================================================================
    // Multimodal Placeholders (100319-100324)
    // =========================================================================

    /// Start of image - image placeholder (100319)
    #[classattr]
    const IMAGE: u32 = 100319;
    /// End of image (100320)
    #[classattr]
    const IMAGE_END: u32 = 100320;
    /// Start of audio - audio placeholder (100321)
    #[classattr]
    const AUDIO: u32 = 100321;
    /// End of audio (100322)
    #[classattr]
    const AUDIO_END: u32 = 100322;
    /// Start of video - video placeholder (100323)
    #[classattr]
    const VIDEO: u32 = 100323;
    /// End of video (100324)
    #[classattr]
    const VIDEO_END: u32 = 100324;

    // =========================================================================
    // Document Structure (100325-100330)
    // =========================================================================

    /// Start of title - document/section title (100325)
    #[classattr]
    const TITLE: u32 = 100325;
    /// End of title (100326)
    #[classattr]
    const TITLE_END: u32 = 100326;
    /// Start of section - semantic document section (100327)
    #[classattr]
    const SECTION: u32 = 100327;
    /// End of section (100328)
    #[classattr]
    const SECTION_END: u32 = 100328;
    /// Start of summary - condensed content summary (100329)
    #[classattr]
    const SUMMARY: u32 = 100329;
    /// End of summary (100330)
    #[classattr]
    const SUMMARY_END: u32 = 100330;

}

/// o200k_base (GPT-4o) Agent Token IDs (200019-200072)
///
/// Access agent token IDs for o200k_base (GPT-4o).
///
/// # Examples
///
/// ```python
/// from splintr import Tokenizer, O200K_AGENT_TOKENS
///
/// tokenizer = Tokenizer.from_pretrained("o200k_base")
/// system_id = O200K_AGENT_TOKENS.SYSTEM  # 200019
/// think_id = O200K_AGENT_TOKENS.THINK   # 200024
///
/// text = "<|system|>You are a helpful assistant"
/// tokens = tokenizer.encode_with_special(text)
/// assert O200K_AGENT_TOKENS.SYSTEM in tokens
/// ```
#[pyclass(name = "O200K_AGENT_TOKENS", frozen)]
pub struct PyO200KAgentTokens;

#[pymethods]
impl PyO200KAgentTokens {
    // =========================================================================
    // Conversation & Roles (200019-200023)
    // =========================================================================

    /// System role - system instructions (200019)
    #[classattr]
    const SYSTEM: u32 = 200019;
    /// User role - user input (200020)
    #[classattr]
    const USER: u32 = 200020;
    /// Assistant role - model output (200021)
    #[classattr]
    const ASSISTANT: u32 = 200021;
    /// Start of message - ChatML wrapper (200022)
    #[classattr]
    const IM_START: u32 = 200022;
    /// End of message - ChatML wrapper (200023)
    #[classattr]
    const IM_END: u32 = 200023;

    // =========================================================================
    // Reasoning/Thinking (200024-200025)
    // =========================================================================

    /// Start of thinking - Chain-of-Thought (200024)
    #[classattr]
    const THINK: u32 = 200024;
    /// End of thinking (200025)
    #[classattr]
    const THINK_END: u32 = 200025;

    // =========================================================================
    // ReAct Agent Loop (200026-200033)
    // =========================================================================

    /// Start of plan - action planning (200026)
    #[classattr]
    const PLAN: u32 = 200026;
    /// End of plan (200027)
    #[classattr]
    const PLAN_END: u32 = 200027;
    /// Start of step - individual action step (200028)
    #[classattr]
    const STEP: u32 = 200028;
    /// End of step (200029)
    #[classattr]
    const STEP_END: u32 = 200029;
    /// Start of action - agent action (200030)
    #[classattr]
    const ACT: u32 = 200030;
    /// End of action (200031)
    #[classattr]
    const ACT_END: u32 = 200031;
    /// Start of observation - environment feedback (200032)
    #[classattr]
    const OBSERVE: u32 = 200032;
    /// End of observation (200033)
    #[classattr]
    const OBSERVE_END: u32 = 200033;

    // =========================================================================
    // Tool/Function Calling (200034-200039)
    // =========================================================================

    /// Start of function call - function invocation (200034)
    #[classattr]
    const FUNCTION: u32 = 200034;
    /// End of function call (200035)
    #[classattr]
    const FUNCTION_END: u32 = 200035;
    /// Start of function result - return value (200036)
    #[classattr]
    const RESULT: u32 = 200036;
    /// End of function result (200037)
    #[classattr]
    const RESULT_END: u32 = 200037;
    /// Start of error - error message (200038)
    #[classattr]
    const ERROR: u32 = 200038;
    /// End of error (200039)
    #[classattr]
    const ERROR_END: u32 = 200039;

    // =========================================================================
    // Code Execution (200040-200045)
    // =========================================================================

    /// Start of code - inline code execution (200040)
    #[classattr]
    const CODE: u32 = 200040;
    /// End of code (200041)
    #[classattr]
    const CODE_END: u32 = 200041;
    /// Start of output - execution output (200042)
    #[classattr]
    const OUTPUT: u32 = 200042;
    /// End of output (200043)
    #[classattr]
    const OUTPUT_END: u32 = 200043;
    /// Start of language tag - code language (200044)
    #[classattr]
    const LANG: u32 = 200044;
    /// End of language tag (200045)
    #[classattr]
    const LANG_END: u32 = 200045;

    // =========================================================================
    // RAG & Citations (200046-200053)
    // =========================================================================

    /// Start of context - retrieved context (200046)
    #[classattr]
    const CONTEXT: u32 = 200046;
    /// End of context (200047)
    #[classattr]
    const CONTEXT_END: u32 = 200047;
    /// Start of quote - exact citation (200048)
    #[classattr]
    const QUOTE: u32 = 200048;
    /// End of quote (200049)
    #[classattr]
    const QUOTE_END: u32 = 200049;
    /// Start of cite - citation reference (200050)
    #[classattr]
    const CITE: u32 = 200050;
    /// End of cite (200051)
    #[classattr]
    const CITE_END: u32 = 200051;
    /// Start of source - document source (200052)
    #[classattr]
    const SOURCE: u32 = 200052;
    /// End of source (200053)
    #[classattr]
    const SOURCE_END: u32 = 200053;

    // =========================================================================
    // Memory/State Management (200054-200057)
    // =========================================================================

    /// Start of memory - persistent memory (200054)
    #[classattr]
    const MEMORY: u32 = 200054;
    /// End of memory (200055)
    #[classattr]
    const MEMORY_END: u32 = 200055;
    /// Start of recall - memory retrieval (200056)
    #[classattr]
    const RECALL: u32 = 200056;
    /// End of recall (200057)
    #[classattr]
    const RECALL_END: u32 = 200057;

    // =========================================================================
    // Control Tokens (200058-200060)
    // =========================================================================

    /// Padding token (200058)
    #[classattr]
    const PAD: u32 = 200058;
    /// Stop generation token (200059)
    #[classattr]
    const STOP: u32 = 200059;
    /// Separator token (200060)
    #[classattr]
    const SEP: u32 = 200060;

    // =========================================================================
    // Multimodal Placeholders (200061-200066)
    // =========================================================================

    /// Start of image - image placeholder (200061)
    #[classattr]
    const IMAGE: u32 = 200061;
    /// End of image (200062)
    #[classattr]
    const IMAGE_END: u32 = 200062;
    /// Start of audio - audio placeholder (200063)
    #[classattr]
    const AUDIO: u32 = 200063;
    /// End of audio (200064)
    #[classattr]
    const AUDIO_END: u32 = 200064;
    /// Start of video - video placeholder (200065)
    #[classattr]
    const VIDEO: u32 = 200065;
    /// End of video (200066)
    #[classattr]
    const VIDEO_END: u32 = 200066;

    // =========================================================================
    // Document Structure (200067-200072)
    // =========================================================================

    /// Start of title - document/section title (200067)
    #[classattr]
    const TITLE: u32 = 200067;
    /// End of title (200068)
    #[classattr]
    const TITLE_END: u32 = 200068;
    /// Start of section - semantic document section (200069)
    #[classattr]
    const SECTION: u32 = 200069;
    /// End of section (200070)
    #[classattr]
    const SECTION_END: u32 = 200070;
    /// Start of summary - condensed content summary (200071)
    #[classattr]
    const SUMMARY: u32 = 200071;
    /// End of summary (200072)
    #[classattr]
    const SUMMARY_END: u32 = 200072;

}

/// Llama 3 family Agent Token IDs (128300-128353)
///
/// Access agent token IDs for Llama 3 family.
///
/// # Examples
///
/// ```python
/// from splintr import Tokenizer, LLAMA3_AGENT_TOKENS
///
/// tokenizer = Tokenizer.from_pretrained("llama3")
/// system_id = LLAMA3_AGENT_TOKENS.SYSTEM  # 128300
/// think_id = LLAMA3_AGENT_TOKENS.THINK   # 128305
///
/// text = "<|system|>You are a helpful assistant"
/// tokens = tokenizer.encode_with_special(text)
/// assert LLAMA3_AGENT_TOKENS.SYSTEM in tokens
/// ```
#[pyclass(name = "LLAMA3_AGENT_TOKENS", frozen)]
pub struct PyLlama3AgentTokens;

#[pymethods]
impl PyLlama3AgentTokens {
    // =========================================================================
    // Model-Specific Native Tokens
    // =========================================================================

    /// Start of text marker (128000)
    #[classattr]
    const BEGIN_OF_TEXT: u32 = 128000;
    /// End of text marker (128001)
    #[classattr]
    const END_OF_TEXT: u32 = 128001;
    /// Padding token for fine-tuning - added in Llama 3.1 (128004)
    #[classattr]
    const FINETUNE_RIGHT_PAD_ID: u32 = 128004;
    /// Step marker for reasoning - added in Llama 3.2-Vision (128005)
    #[classattr]
    const STEP_ID: u32 = 128005;
    /// Start of header marker (128006)
    #[classattr]
    const START_HEADER_ID: u32 = 128006;
    /// End of header marker (128007)
    #[classattr]
    const END_HEADER_ID: u32 = 128007;
    /// End of message marker for tool use - added in Llama 3.1 (128008)
    #[classattr]
    const EOM_ID: u32 = 128008;
    /// End of turn marker (128009)
    #[classattr]
    const EOT_ID: u32 = 128009;
    /// Python code interpreter marker - added in Llama 3.1 (128010)
    #[classattr]
    const PYTHON_TAG: u32 = 128010;
    /// Image placeholder - Llama 3.2-Vision official (128256)
    #[classattr]
    const IMAGE: u32 = 128256;
    /// End of image placeholder (128257)
    #[classattr]
    const IMAGE_END: u32 = 128257;
    /// Audio placeholder (128258)
    #[classattr]
    const AUDIO: u32 = 128258;
    /// End of audio placeholder (128259)
    #[classattr]
    const AUDIO_END: u32 = 128259;
    /// Video placeholder (128260)
    #[classattr]
    const VIDEO: u32 = 128260;
    /// End of video placeholder (128261)
    #[classattr]
    const VIDEO_END: u32 = 128261;

    // =========================================================================
    // Conversation & Roles (128300-128304)
    // =========================================================================

    /// System role - system instructions (128300)
    #[classattr]
    const SYSTEM: u32 = 128300;
    /// User role - user input (128301)
    #[classattr]
    const USER: u32 = 128301;
    /// Assistant role - model output (128302)
    #[classattr]
    const ASSISTANT: u32 = 128302;
    /// Start of message - ChatML wrapper (128303)
    #[classattr]
    const IM_START: u32 = 128303;
    /// End of message - ChatML wrapper (128304)
    #[classattr]
    const IM_END: u32 = 128304;

    // =========================================================================
    // Reasoning/Thinking (128305-128306)
    // =========================================================================

    /// Start of thinking - Chain-of-Thought (128305)
    #[classattr]
    const THINK: u32 = 128305;
    /// End of thinking (128306)
    #[classattr]
    const THINK_END: u32 = 128306;

    // =========================================================================
    // ReAct Agent Loop (128307-128314)
    // =========================================================================

    /// Start of plan - action planning (128307)
    #[classattr]
    const PLAN: u32 = 128307;
    /// End of plan (128308)
    #[classattr]
    const PLAN_END: u32 = 128308;
    /// Start of step - individual action step (128309)
    #[classattr]
    const STEP: u32 = 128309;
    /// End of step (128310)
    #[classattr]
    const STEP_END: u32 = 128310;
    /// Start of action - agent action (128311)
    #[classattr]
    const ACT: u32 = 128311;
    /// End of action (128312)
    #[classattr]
    const ACT_END: u32 = 128312;
    /// Start of observation - environment feedback (128313)
    #[classattr]
    const OBSERVE: u32 = 128313;
    /// End of observation (128314)
    #[classattr]
    const OBSERVE_END: u32 = 128314;

    // =========================================================================
    // Tool/Function Calling (128315-128320)
    // =========================================================================

    /// Start of function call - function invocation (128315)
    #[classattr]
    const FUNCTION: u32 = 128315;
    /// End of function call (128316)
    #[classattr]
    const FUNCTION_END: u32 = 128316;
    /// Start of function result - return value (128317)
    #[classattr]
    const RESULT: u32 = 128317;
    /// End of function result (128318)
    #[classattr]
    const RESULT_END: u32 = 128318;
    /// Start of error - error message (128319)
    #[classattr]
    const ERROR: u32 = 128319;
    /// End of error (128320)
    #[classattr]
    const ERROR_END: u32 = 128320;

    // =========================================================================
    // Code Execution (128321-128326)
    // =========================================================================

    /// Start of code - inline code execution (128321)
    #[classattr]
    const CODE: u32 = 128321;
    /// End of code (128322)
    #[classattr]
    const CODE_END: u32 = 128322;
    /// Start of output - execution output (128323)
    #[classattr]
    const OUTPUT: u32 = 128323;
    /// End of output (128324)
    #[classattr]
    const OUTPUT_END: u32 = 128324;
    /// Start of language tag - code language (128325)
    #[classattr]
    const LANG: u32 = 128325;
    /// End of language tag (128326)
    #[classattr]
    const LANG_END: u32 = 128326;

    // =========================================================================
    // RAG & Citations (128327-128334)
    // =========================================================================

    /// Start of context - retrieved context (128327)
    #[classattr]
    const CONTEXT: u32 = 128327;
    /// End of context (128328)
    #[classattr]
    const CONTEXT_END: u32 = 128328;
    /// Start of quote - exact citation (128329)
    #[classattr]
    const QUOTE: u32 = 128329;
    /// End of quote (128330)
    #[classattr]
    const QUOTE_END: u32 = 128330;
    /// Start of cite - citation reference (128331)
    #[classattr]
    const CITE: u32 = 128331;
    /// End of cite (128332)
    #[classattr]
    const CITE_END: u32 = 128332;
    /// Start of source - document source (128333)
    #[classattr]
    const SOURCE: u32 = 128333;
    /// End of source (128334)
    #[classattr]
    const SOURCE_END: u32 = 128334;

    // =========================================================================
    // Memory/State Management (128335-128338)
    // =========================================================================

    /// Start of memory - persistent memory (128335)
    #[classattr]
    const MEMORY: u32 = 128335;
    /// End of memory (128336)
    #[classattr]
    const MEMORY_END: u32 = 128336;
    /// Start of recall - memory retrieval (128337)
    #[classattr]
    const RECALL: u32 = 128337;
    /// End of recall (128338)
    #[classattr]
    const RECALL_END: u32 = 128338;

    // =========================================================================
    // Control Tokens (128339-128341)
    // =========================================================================

    /// Padding token (128339)
    #[classattr]
    const PAD: u32 = 128339;
    /// Stop generation token (128340)
    #[classattr]
    const STOP: u32 = 128340;
    /// Separator token (128341)
    #[classattr]
    const SEP: u32 = 128341;

    // =========================================================================
    // Multimodal Placeholders (128342-128347)
    // =========================================================================


    // =========================================================================
    // Document Structure (128348-128353)
    // =========================================================================

    /// Start of title - document/section title (128348)
    #[classattr]
    const TITLE: u32 = 128348;
    /// End of title (128349)
    #[classattr]
    const TITLE_END: u32 = 128349;
    /// Start of section - semantic document section (128350)
    #[classattr]
    const SECTION: u32 = 128350;
    /// End of section (128351)
    #[classattr]
    const SECTION_END: u32 = 128351;
    /// Start of summary - condensed content summary (128352)
    #[classattr]
    const SUMMARY: u32 = 128352;
    /// End of summary (128353)
    #[classattr]
    const SUMMARY_END: u32 = 128353;

}

/// DeepSeek V3/R1 Agent Token IDs (128900-128953)
///
/// Access agent token IDs for DeepSeek V3/R1.
///
/// # Examples
///
/// ```python
/// from splintr import Tokenizer, DEEPSEEK_V3_AGENT_TOKENS
///
/// tokenizer = Tokenizer.from_pretrained("deepseek_v3")
/// system_id = DEEPSEEK_V3_AGENT_TOKENS.SYSTEM  # 128900
/// think_id = DEEPSEEK_V3_AGENT_TOKENS.THINK   # 128905
///
/// text = "<|system|>You are a helpful assistant"
/// tokens = tokenizer.encode_with_special(text)
/// assert DEEPSEEK_V3_AGENT_TOKENS.SYSTEM in tokens
/// ```
#[pyclass(name = "DEEPSEEK_V3_AGENT_TOKENS", frozen)]
pub struct PyDeepSeekV3AgentTokens;

#[pymethods]
impl PyDeepSeekV3AgentTokens {
    // =========================================================================
    // Model-Specific Native Tokens
    // =========================================================================

    /// Begin of sentence marker (0)
    #[classattr]
    const BEGIN_OF_SENTENCE: u32 = 0;
    /// End of sentence marker (1)
    #[classattr]
    const END_OF_SENTENCE: u32 = 1;
    /// Native padding token (2)
    #[classattr]
    const PAD_NATIVE: u32 = 2;
    /// Native <think> token (128798)
    #[classattr]
    const THINK_NATIVE: u32 = 128798;
    /// Native </think> token (128799)
    #[classattr]
    const THINK_END_NATIVE: u32 = 128799;
    /// Fill-in-the-Middle hole (128800)
    #[classattr]
    const FIM_HOLE: u32 = 128800;
    /// Fill-in-the-Middle begin (128801)
    #[classattr]
    const FIM_BEGIN: u32 = 128801;
    /// Fill-in-the-Middle end (128802)
    #[classattr]
    const FIM_END: u32 = 128802;
    /// Native <｜User｜> token (128803)
    #[classattr]
    const USER_NATIVE: u32 = 128803;
    /// Native <｜Assistant｜> token (128804)
    #[classattr]
    const ASSISTANT_NATIVE: u32 = 128804;
    /// End of turn <|EOT|> (128805)
    #[classattr]
    const EOT: u32 = 128805;
    /// Tool calls begin marker (128806)
    #[classattr]
    const TOOL_CALLS_BEGIN: u32 = 128806;
    /// Tool calls end marker (128807)
    #[classattr]
    const TOOL_CALLS_END: u32 = 128807;
    /// Single tool call begin (128808)
    #[classattr]
    const TOOL_CALL_BEGIN: u32 = 128808;
    /// Single tool call end (128809)
    #[classattr]
    const TOOL_CALL_END: u32 = 128809;
    /// Tool outputs begin marker (128810)
    #[classattr]
    const TOOL_OUTPUTS_BEGIN: u32 = 128810;
    /// Tool outputs end marker (128811)
    #[classattr]
    const TOOL_OUTPUTS_END: u32 = 128811;
    /// Single tool output begin (128812)
    #[classattr]
    const TOOL_OUTPUT_BEGIN: u32 = 128812;
    /// Single tool output end (128813)
    #[classattr]
    const TOOL_OUTPUT_END: u32 = 128813;
    /// Tool separator (128814)
    #[classattr]
    const TOOL_SEP: u32 = 128814;

    // =========================================================================
    // Conversation & Roles (128900-128904)
    // =========================================================================

    /// System role - system instructions (128900)
    #[classattr]
    const SYSTEM: u32 = 128900;
    /// User role - user input (128901)
    #[classattr]
    const USER: u32 = 128901;
    /// Assistant role - model output (128902)
    #[classattr]
    const ASSISTANT: u32 = 128902;
    /// Start of message - ChatML wrapper (128903)
    #[classattr]
    const IM_START: u32 = 128903;
    /// End of message - ChatML wrapper (128904)
    #[classattr]
    const IM_END: u32 = 128904;

    // =========================================================================
    // Reasoning/Thinking (128905-128906)
    // =========================================================================

    /// Start of thinking - Chain-of-Thought (128905)
    #[classattr]
    const THINK: u32 = 128905;
    /// End of thinking (128906)
    #[classattr]
    const THINK_END: u32 = 128906;

    // =========================================================================
    // ReAct Agent Loop (128907-128914)
    // =========================================================================

    /// Start of plan - action planning (128907)
    #[classattr]
    const PLAN: u32 = 128907;
    /// End of plan (128908)
    #[classattr]
    const PLAN_END: u32 = 128908;
    /// Start of step - individual action step (128909)
    #[classattr]
    const STEP: u32 = 128909;
    /// End of step (128910)
    #[classattr]
    const STEP_END: u32 = 128910;
    /// Start of action - agent action (128911)
    #[classattr]
    const ACT: u32 = 128911;
    /// End of action (128912)
    #[classattr]
    const ACT_END: u32 = 128912;
    /// Start of observation - environment feedback (128913)
    #[classattr]
    const OBSERVE: u32 = 128913;
    /// End of observation (128914)
    #[classattr]
    const OBSERVE_END: u32 = 128914;

    // =========================================================================
    // Tool/Function Calling (128915-128920)
    // =========================================================================

    /// Start of function call - function invocation (128915)
    #[classattr]
    const FUNCTION: u32 = 128915;
    /// End of function call (128916)
    #[classattr]
    const FUNCTION_END: u32 = 128916;
    /// Start of function result - return value (128917)
    #[classattr]
    const RESULT: u32 = 128917;
    /// End of function result (128918)
    #[classattr]
    const RESULT_END: u32 = 128918;
    /// Start of error - error message (128919)
    #[classattr]
    const ERROR: u32 = 128919;
    /// End of error (128920)
    #[classattr]
    const ERROR_END: u32 = 128920;

    // =========================================================================
    // Code Execution (128921-128926)
    // =========================================================================

    /// Start of code - inline code execution (128921)
    #[classattr]
    const CODE: u32 = 128921;
    /// End of code (128922)
    #[classattr]
    const CODE_END: u32 = 128922;
    /// Start of output - execution output (128923)
    #[classattr]
    const OUTPUT: u32 = 128923;
    /// End of output (128924)
    #[classattr]
    const OUTPUT_END: u32 = 128924;
    /// Start of language tag - code language (128925)
    #[classattr]
    const LANG: u32 = 128925;
    /// End of language tag (128926)
    #[classattr]
    const LANG_END: u32 = 128926;

    // =========================================================================
    // RAG & Citations (128927-128934)
    // =========================================================================

    /// Start of context - retrieved context (128927)
    #[classattr]
    const CONTEXT: u32 = 128927;
    /// End of context (128928)
    #[classattr]
    const CONTEXT_END: u32 = 128928;
    /// Start of quote - exact citation (128929)
    #[classattr]
    const QUOTE: u32 = 128929;
    /// End of quote (128930)
    #[classattr]
    const QUOTE_END: u32 = 128930;
    /// Start of cite - citation reference (128931)
    #[classattr]
    const CITE: u32 = 128931;
    /// End of cite (128932)
    #[classattr]
    const CITE_END: u32 = 128932;
    /// Start of source - document source (128933)
    #[classattr]
    const SOURCE: u32 = 128933;
    /// End of source (128934)
    #[classattr]
    const SOURCE_END: u32 = 128934;

    // =========================================================================
    // Memory/State Management (128935-128938)
    // =========================================================================

    /// Start of memory - persistent memory (128935)
    #[classattr]
    const MEMORY: u32 = 128935;
    /// End of memory (128936)
    #[classattr]
    const MEMORY_END: u32 = 128936;
    /// Start of recall - memory retrieval (128937)
    #[classattr]
    const RECALL: u32 = 128937;
    /// End of recall (128938)
    #[classattr]
    const RECALL_END: u32 = 128938;

    // =========================================================================
    // Control Tokens (128939-128941)
    // =========================================================================

    /// Padding token (128939)
    #[classattr]
    const PAD: u32 = 128939;
    /// Stop generation token (128940)
    #[classattr]
    const STOP: u32 = 128940;
    /// Separator token (128941)
    #[classattr]
    const SEP: u32 = 128941;

    // =========================================================================
    // Multimodal Placeholders (128942-128947)
    // =========================================================================

    /// Start of image - image placeholder (128942)
    #[classattr]
    const IMAGE: u32 = 128942;
    /// End of image (128943)
    #[classattr]
    const IMAGE_END: u32 = 128943;
    /// Start of audio - audio placeholder (128944)
    #[classattr]
    const AUDIO: u32 = 128944;
    /// End of audio (128945)
    #[classattr]
    const AUDIO_END: u32 = 128945;
    /// Start of video - video placeholder (128946)
    #[classattr]
    const VIDEO: u32 = 128946;
    /// End of video (128947)
    #[classattr]
    const VIDEO_END: u32 = 128947;

    // =========================================================================
    // Document Structure (128948-128953)
    // =========================================================================

    /// Start of title - document/section title (128948)
    #[classattr]
    const TITLE: u32 = 128948;
    /// End of title (128949)
    #[classattr]
    const TITLE_END: u32 = 128949;
    /// Start of section - semantic document section (128950)
    #[classattr]
    const SECTION: u32 = 128950;
    /// End of section (128951)
    #[classattr]
    const SECTION_END: u32 = 128951;
    /// Start of summary - condensed content summary (128952)
    #[classattr]
    const SUMMARY: u32 = 128952;
    /// End of summary (128953)
    #[classattr]
    const SUMMARY_END: u32 = 128953;

}

/// Mistral V1 (7B v0.1/v0.2, Mixtral 8x7B) Agent Token IDs (32000-32053)
///
/// Access agent token IDs for Mistral V1 (7B v0.1/v0.2, Mixtral 8x7B).
///
/// # Examples
///
/// ```python
/// from splintr import Tokenizer, MISTRAL_V1_AGENT_TOKENS
///
/// tokenizer = Tokenizer.from_pretrained("mistral_v1")
/// system_id = MISTRAL_V1_AGENT_TOKENS.SYSTEM  # 32000
/// think_id = MISTRAL_V1_AGENT_TOKENS.THINK   # 32005
///
/// text = "<|system|>You are a helpful assistant"
/// tokens = tokenizer.encode_with_special(text)
/// assert MISTRAL_V1_AGENT_TOKENS.SYSTEM in tokens
/// ```
#[pyclass(name = "MISTRAL_V1_AGENT_TOKENS", frozen)]
pub struct PyMistralV1AgentTokens;

#[pymethods]
impl PyMistralV1AgentTokens {
    // =========================================================================
    // Conversation & Roles (32000-32004)
    // =========================================================================

    /// System role - system instructions (32000)
    #[classattr]
    const SYSTEM: u32 = 32000;
    /// User role - user input (32001)
    #[classattr]
    const USER: u32 = 32001;
    /// Assistant role - model output (32002)
    #[classattr]
    const ASSISTANT: u32 = 32002;
    /// Start of message - ChatML wrapper (32003)
    #[classattr]
    const IM_START: u32 = 32003;
    /// End of message - ChatML wrapper (32004)
    #[classattr]
    const IM_END: u32 = 32004;

    // =========================================================================
    // Reasoning/Thinking (32005-32006)
    // =========================================================================

    /// Start of thinking - Chain-of-Thought (32005)
    #[classattr]
    const THINK: u32 = 32005;
    /// End of thinking (32006)
    #[classattr]
    const THINK_END: u32 = 32006;

    // =========================================================================
    // ReAct Agent Loop (32007-32014)
    // =========================================================================

    /// Start of plan - action planning (32007)
    #[classattr]
    const PLAN: u32 = 32007;
    /// End of plan (32008)
    #[classattr]
    const PLAN_END: u32 = 32008;
    /// Start of step - individual action step (32009)
    #[classattr]
    const STEP: u32 = 32009;
    /// End of step (32010)
    #[classattr]
    const STEP_END: u32 = 32010;
    /// Start of action - agent action (32011)
    #[classattr]
    const ACT: u32 = 32011;
    /// End of action (32012)
    #[classattr]
    const ACT_END: u32 = 32012;
    /// Start of observation - environment feedback (32013)
    #[classattr]
    const OBSERVE: u32 = 32013;
    /// End of observation (32014)
    #[classattr]
    const OBSERVE_END: u32 = 32014;

    // =========================================================================
    // Tool/Function Calling (32015-32020)
    // =========================================================================

    /// Start of function call - function invocation (32015)
    #[classattr]
    const FUNCTION: u32 = 32015;
    /// End of function call (32016)
    #[classattr]
    const FUNCTION_END: u32 = 32016;
    /// Start of function result - return value (32017)
    #[classattr]
    const RESULT: u32 = 32017;
    /// End of function result (32018)
    #[classattr]
    const RESULT_END: u32 = 32018;
    /// Start of error - error message (32019)
    #[classattr]
    const ERROR: u32 = 32019;
    /// End of error (32020)
    #[classattr]
    const ERROR_END: u32 = 32020;

    // =========================================================================
    // Code Execution (32021-32026)
    // =========================================================================

    /// Start of code - inline code execution (32021)
    #[classattr]
    const CODE: u32 = 32021;
    /// End of code (32022)
    #[classattr]
    const CODE_END: u32 = 32022;
    /// Start of output - execution output (32023)
    #[classattr]
    const OUTPUT: u32 = 32023;
    /// End of output (32024)
    #[classattr]
    const OUTPUT_END: u32 = 32024;
    /// Start of language tag - code language (32025)
    #[classattr]
    const LANG: u32 = 32025;
    /// End of language tag (32026)
    #[classattr]
    const LANG_END: u32 = 32026;

    // =========================================================================
    // RAG & Citations (32027-32034)
    // =========================================================================

    /// Start of context - retrieved context (32027)
    #[classattr]
    const CONTEXT: u32 = 32027;
    /// End of context (32028)
    #[classattr]
    const CONTEXT_END: u32 = 32028;
    /// Start of quote - exact citation (32029)
    #[classattr]
    const QUOTE: u32 = 32029;
    /// End of quote (32030)
    #[classattr]
    const QUOTE_END: u32 = 32030;
    /// Start of cite - citation reference (32031)
    #[classattr]
    const CITE: u32 = 32031;
    /// End of cite (32032)
    #[classattr]
    const CITE_END: u32 = 32032;
    /// Start of source - document source (32033)
    #[classattr]
    const SOURCE: u32 = 32033;
    /// End of source (32034)
    #[classattr]
    const SOURCE_END: u32 = 32034;

    // =========================================================================
    // Memory/State Management (32035-32038)
    // =========================================================================

    /// Start of memory - persistent memory (32035)
    #[classattr]
    const MEMORY: u32 = 32035;
    /// End of memory (32036)
    #[classattr]
    const MEMORY_END: u32 = 32036;
    /// Start of recall - memory retrieval (32037)
    #[classattr]
    const RECALL: u32 = 32037;
    /// End of recall (32038)
    #[classattr]
    const RECALL_END: u32 = 32038;

    // =========================================================================
    // Control Tokens (32039-32041)
    // =========================================================================

    /// Padding token (32039)
    #[classattr]
    const PAD: u32 = 32039;
    /// Stop generation token (32040)
    #[classattr]
    const STOP: u32 = 32040;
    /// Separator token (32041)
    #[classattr]
    const SEP: u32 = 32041;

    // =========================================================================
    // Multimodal Placeholders (32042-32047)
    // =========================================================================

    /// Start of image - image placeholder (32042)
    #[classattr]
    const IMAGE: u32 = 32042;
    /// End of image (32043)
    #[classattr]
    const IMAGE_END: u32 = 32043;
    /// Start of audio - audio placeholder (32044)
    #[classattr]
    const AUDIO: u32 = 32044;
    /// End of audio (32045)
    #[classattr]
    const AUDIO_END: u32 = 32045;
    /// Start of video - video placeholder (32046)
    #[classattr]
    const VIDEO: u32 = 32046;
    /// End of video (32047)
    #[classattr]
    const VIDEO_END: u32 = 32047;

    // =========================================================================
    // Document Structure (32048-32053)
    // =========================================================================

    /// Start of title - document/section title (32048)
    #[classattr]
    const TITLE: u32 = 32048;
    /// End of title (32049)
    #[classattr]
    const TITLE_END: u32 = 32049;
    /// Start of section - semantic document section (32050)
    #[classattr]
    const SECTION: u32 = 32050;
    /// End of section (32051)
    #[classattr]
    const SECTION_END: u32 = 32051;
    /// Start of summary - condensed content summary (32052)
    #[classattr]
    const SUMMARY: u32 = 32052;
    /// End of summary (32053)
    #[classattr]
    const SUMMARY_END: u32 = 32053;

}

/// Mistral V2 (7B v0.3, Mixtral 8x22B, Codestral) Agent Token IDs (32768-32821)
///
/// Access agent token IDs for Mistral V2 (7B v0.3, Mixtral 8x22B, Codestral).
///
/// # Examples
///
/// ```python
/// from splintr import Tokenizer, MISTRAL_V2_AGENT_TOKENS
///
/// tokenizer = Tokenizer.from_pretrained("mistral_v2")
/// system_id = MISTRAL_V2_AGENT_TOKENS.SYSTEM  # 32768
/// think_id = MISTRAL_V2_AGENT_TOKENS.THINK   # 32773
///
/// text = "<|system|>You are a helpful assistant"
/// tokens = tokenizer.encode_with_special(text)
/// assert MISTRAL_V2_AGENT_TOKENS.SYSTEM in tokens
/// ```
#[pyclass(name = "MISTRAL_V2_AGENT_TOKENS", frozen)]
pub struct PyMistralV2AgentTokens;

#[pymethods]
impl PyMistralV2AgentTokens {
    // =========================================================================
    // Conversation & Roles (32768-32772)
    // =========================================================================

    /// System role - system instructions (32768)
    #[classattr]
    const SYSTEM: u32 = 32768;
    /// User role - user input (32769)
    #[classattr]
    const USER: u32 = 32769;
    /// Assistant role - model output (32770)
    #[classattr]
    const ASSISTANT: u32 = 32770;
    /// Start of message - ChatML wrapper (32771)
    #[classattr]
    const IM_START: u32 = 32771;
    /// End of message - ChatML wrapper (32772)
    #[classattr]
    const IM_END: u32 = 32772;

    // =========================================================================
    // Reasoning/Thinking (32773-32774)
    // =========================================================================

    /// Start of thinking - Chain-of-Thought (32773)
    #[classattr]
    const THINK: u32 = 32773;
    /// End of thinking (32774)
    #[classattr]
    const THINK_END: u32 = 32774;

    // =========================================================================
    // ReAct Agent Loop (32775-32782)
    // =========================================================================

    /// Start of plan - action planning (32775)
    #[classattr]
    const PLAN: u32 = 32775;
    /// End of plan (32776)
    #[classattr]
    const PLAN_END: u32 = 32776;
    /// Start of step - individual action step (32777)
    #[classattr]
    const STEP: u32 = 32777;
    /// End of step (32778)
    #[classattr]
    const STEP_END: u32 = 32778;
    /// Start of action - agent action (32779)
    #[classattr]
    const ACT: u32 = 32779;
    /// End of action (32780)
    #[classattr]
    const ACT_END: u32 = 32780;
    /// Start of observation - environment feedback (32781)
    #[classattr]
    const OBSERVE: u32 = 32781;
    /// End of observation (32782)
    #[classattr]
    const OBSERVE_END: u32 = 32782;

    // =========================================================================
    // Tool/Function Calling (32783-32788)
    // =========================================================================

    /// Start of function call - function invocation (32783)
    #[classattr]
    const FUNCTION: u32 = 32783;
    /// End of function call (32784)
    #[classattr]
    const FUNCTION_END: u32 = 32784;
    /// Start of function result - return value (32785)
    #[classattr]
    const RESULT: u32 = 32785;
    /// End of function result (32786)
    #[classattr]
    const RESULT_END: u32 = 32786;
    /// Start of error - error message (32787)
    #[classattr]
    const ERROR: u32 = 32787;
    /// End of error (32788)
    #[classattr]
    const ERROR_END: u32 = 32788;

    // =========================================================================
    // Code Execution (32789-32794)
    // =========================================================================

    /// Start of code - inline code execution (32789)
    #[classattr]
    const CODE: u32 = 32789;
    /// End of code (32790)
    #[classattr]
    const CODE_END: u32 = 32790;
    /// Start of output - execution output (32791)
    #[classattr]
    const OUTPUT: u32 = 32791;
    /// End of output (32792)
    #[classattr]
    const OUTPUT_END: u32 = 32792;
    /// Start of language tag - code language (32793)
    #[classattr]
    const LANG: u32 = 32793;
    /// End of language tag (32794)
    #[classattr]
    const LANG_END: u32 = 32794;

    // =========================================================================
    // RAG & Citations (32795-32802)
    // =========================================================================

    /// Start of context - retrieved context (32795)
    #[classattr]
    const CONTEXT: u32 = 32795;
    /// End of context (32796)
    #[classattr]
    const CONTEXT_END: u32 = 32796;
    /// Start of quote - exact citation (32797)
    #[classattr]
    const QUOTE: u32 = 32797;
    /// End of quote (32798)
    #[classattr]
    const QUOTE_END: u32 = 32798;
    /// Start of cite - citation reference (32799)
    #[classattr]
    const CITE: u32 = 32799;
    /// End of cite (32800)
    #[classattr]
    const CITE_END: u32 = 32800;
    /// Start of source - document source (32801)
    #[classattr]
    const SOURCE: u32 = 32801;
    /// End of source (32802)
    #[classattr]
    const SOURCE_END: u32 = 32802;

    // =========================================================================
    // Memory/State Management (32803-32806)
    // =========================================================================

    /// Start of memory - persistent memory (32803)
    #[classattr]
    const MEMORY: u32 = 32803;
    /// End of memory (32804)
    #[classattr]
    const MEMORY_END: u32 = 32804;
    /// Start of recall - memory retrieval (32805)
    #[classattr]
    const RECALL: u32 = 32805;
    /// End of recall (32806)
    #[classattr]
    const RECALL_END: u32 = 32806;

    // =========================================================================
    // Control Tokens (32807-32809)
    // =========================================================================

    /// Padding token (32807)
    #[classattr]
    const PAD: u32 = 32807;
    /// Stop generation token (32808)
    #[classattr]
    const STOP: u32 = 32808;
    /// Separator token (32809)
    #[classattr]
    const SEP: u32 = 32809;

    // =========================================================================
    // Multimodal Placeholders (32810-32815)
    // =========================================================================

    /// Start of image - image placeholder (32810)
    #[classattr]
    const IMAGE: u32 = 32810;
    /// End of image (32811)
    #[classattr]
    const IMAGE_END: u32 = 32811;
    /// Start of audio - audio placeholder (32812)
    #[classattr]
    const AUDIO: u32 = 32812;
    /// End of audio (32813)
    #[classattr]
    const AUDIO_END: u32 = 32813;
    /// Start of video - video placeholder (32814)
    #[classattr]
    const VIDEO: u32 = 32814;
    /// End of video (32815)
    #[classattr]
    const VIDEO_END: u32 = 32815;

    // =========================================================================
    // Document Structure (32816-32821)
    // =========================================================================

    /// Start of title - document/section title (32816)
    #[classattr]
    const TITLE: u32 = 32816;
    /// End of title (32817)
    #[classattr]
    const TITLE_END: u32 = 32817;
    /// Start of section - semantic document section (32818)
    #[classattr]
    const SECTION: u32 = 32818;
    /// End of section (32819)
    #[classattr]
    const SECTION_END: u32 = 32819;
    /// Start of summary - condensed content summary (32820)
    #[classattr]
    const SUMMARY: u32 = 32820;
    /// End of summary (32821)
    #[classattr]
    const SUMMARY_END: u32 = 32821;

}

/// Mistral V3/Tekken (NeMo, Large 2, Pixtral) Agent Token IDs (131072-131125)
///
/// Access agent token IDs for Mistral V3/Tekken (NeMo, Large 2, Pixtral).
///
/// # Examples
///
/// ```python
/// from splintr import Tokenizer, MISTRAL_V3_AGENT_TOKENS
///
/// tokenizer = Tokenizer.from_pretrained("mistral_v3")
/// system_id = MISTRAL_V3_AGENT_TOKENS.SYSTEM  # 131072
/// think_id = MISTRAL_V3_AGENT_TOKENS.THINK   # 131077
///
/// text = "<|system|>You are a helpful assistant"
/// tokens = tokenizer.encode_with_special(text)
/// assert MISTRAL_V3_AGENT_TOKENS.SYSTEM in tokens
/// ```
#[pyclass(name = "MISTRAL_V3_AGENT_TOKENS", frozen)]
pub struct PyMistralV3AgentTokens;

#[pymethods]
impl PyMistralV3AgentTokens {
    // =========================================================================
    // Model-Specific Native Tokens
    // =========================================================================

    /// [INST] instruction begin (3)
    #[classattr]
    const INST: u32 = 3;
    /// [/INST] instruction end (4)
    #[classattr]
    const INST_END: u32 = 4;
    /// [AVAILABLE_TOOLS] tool list begin (5)
    #[classattr]
    const AVAILABLE_TOOLS: u32 = 5;
    /// [/AVAILABLE_TOOLS] tool list end (6)
    #[classattr]
    const AVAILABLE_TOOLS_END: u32 = 6;
    /// [TOOL_RESULTS] tool results begin (7)
    #[classattr]
    const TOOL_RESULTS: u32 = 7;
    /// [/TOOL_RESULTS] tool results end (8)
    #[classattr]
    const TOOL_RESULTS_END: u32 = 8;
    /// [TOOL_CALLS] tool calls marker (9)
    #[classattr]
    const TOOL_CALLS: u32 = 9;

    // =========================================================================
    // Conversation & Roles (131072-131076)
    // =========================================================================

    /// System role - system instructions (131072)
    #[classattr]
    const SYSTEM: u32 = 131072;
    /// User role - user input (131073)
    #[classattr]
    const USER: u32 = 131073;
    /// Assistant role - model output (131074)
    #[classattr]
    const ASSISTANT: u32 = 131074;
    /// Start of message - ChatML wrapper (131075)
    #[classattr]
    const IM_START: u32 = 131075;
    /// End of message - ChatML wrapper (131076)
    #[classattr]
    const IM_END: u32 = 131076;

    // =========================================================================
    // Reasoning/Thinking (131077-131078)
    // =========================================================================

    /// Start of thinking - Chain-of-Thought (131077)
    #[classattr]
    const THINK: u32 = 131077;
    /// End of thinking (131078)
    #[classattr]
    const THINK_END: u32 = 131078;

    // =========================================================================
    // ReAct Agent Loop (131079-131086)
    // =========================================================================

    /// Start of plan - action planning (131079)
    #[classattr]
    const PLAN: u32 = 131079;
    /// End of plan (131080)
    #[classattr]
    const PLAN_END: u32 = 131080;
    /// Start of step - individual action step (131081)
    #[classattr]
    const STEP: u32 = 131081;
    /// End of step (131082)
    #[classattr]
    const STEP_END: u32 = 131082;
    /// Start of action - agent action (131083)
    #[classattr]
    const ACT: u32 = 131083;
    /// End of action (131084)
    #[classattr]
    const ACT_END: u32 = 131084;
    /// Start of observation - environment feedback (131085)
    #[classattr]
    const OBSERVE: u32 = 131085;
    /// End of observation (131086)
    #[classattr]
    const OBSERVE_END: u32 = 131086;

    // =========================================================================
    // Tool/Function Calling (131087-131092)
    // =========================================================================

    /// Start of function call - function invocation (131087)
    #[classattr]
    const FUNCTION: u32 = 131087;
    /// End of function call (131088)
    #[classattr]
    const FUNCTION_END: u32 = 131088;
    /// Start of function result - return value (131089)
    #[classattr]
    const RESULT: u32 = 131089;
    /// End of function result (131090)
    #[classattr]
    const RESULT_END: u32 = 131090;
    /// Start of error - error message (131091)
    #[classattr]
    const ERROR: u32 = 131091;
    /// End of error (131092)
    #[classattr]
    const ERROR_END: u32 = 131092;

    // =========================================================================
    // Code Execution (131093-131098)
    // =========================================================================

    /// Start of code - inline code execution (131093)
    #[classattr]
    const CODE: u32 = 131093;
    /// End of code (131094)
    #[classattr]
    const CODE_END: u32 = 131094;
    /// Start of output - execution output (131095)
    #[classattr]
    const OUTPUT: u32 = 131095;
    /// End of output (131096)
    #[classattr]
    const OUTPUT_END: u32 = 131096;
    /// Start of language tag - code language (131097)
    #[classattr]
    const LANG: u32 = 131097;
    /// End of language tag (131098)
    #[classattr]
    const LANG_END: u32 = 131098;

    // =========================================================================
    // RAG & Citations (131099-131106)
    // =========================================================================

    /// Start of context - retrieved context (131099)
    #[classattr]
    const CONTEXT: u32 = 131099;
    /// End of context (131100)
    #[classattr]
    const CONTEXT_END: u32 = 131100;
    /// Start of quote - exact citation (131101)
    #[classattr]
    const QUOTE: u32 = 131101;
    /// End of quote (131102)
    #[classattr]
    const QUOTE_END: u32 = 131102;
    /// Start of cite - citation reference (131103)
    #[classattr]
    const CITE: u32 = 131103;
    /// End of cite (131104)
    #[classattr]
    const CITE_END: u32 = 131104;
    /// Start of source - document source (131105)
    #[classattr]
    const SOURCE: u32 = 131105;
    /// End of source (131106)
    #[classattr]
    const SOURCE_END: u32 = 131106;

    // =========================================================================
    // Memory/State Management (131107-131110)
    // =========================================================================

    /// Start of memory - persistent memory (131107)
    #[classattr]
    const MEMORY: u32 = 131107;
    /// End of memory (131108)
    #[classattr]
    const MEMORY_END: u32 = 131108;
    /// Start of recall - memory retrieval (131109)
    #[classattr]
    const RECALL: u32 = 131109;
    /// End of recall (131110)
    #[classattr]
    const RECALL_END: u32 = 131110;

    // =========================================================================
    // Control Tokens (131111-131113)
    // =========================================================================

    /// Padding token (131111)
    #[classattr]
    const PAD: u32 = 131111;
    /// Stop generation token (131112)
    #[classattr]
    const STOP: u32 = 131112;
    /// Separator token (131113)
    #[classattr]
    const SEP: u32 = 131113;

    // =========================================================================
    // Multimodal Placeholders (131114-131119)
    // =========================================================================

    /// Start of image - image placeholder (131114)
    #[classattr]
    const IMAGE: u32 = 131114;
    /// End of image (131115)
    #[classattr]
    const IMAGE_END: u32 = 131115;
    /// Start of audio - audio placeholder (131116)
    #[classattr]
    const AUDIO: u32 = 131116;
    /// End of audio (131117)
    #[classattr]
    const AUDIO_END: u32 = 131117;
    /// Start of video - video placeholder (131118)
    #[classattr]
    const VIDEO: u32 = 131118;
    /// End of video (131119)
    #[classattr]
    const VIDEO_END: u32 = 131119;

    // =========================================================================
    // Document Structure (131120-131125)
    // =========================================================================

    /// Start of title - document/section title (131120)
    #[classattr]
    const TITLE: u32 = 131120;
    /// End of title (131121)
    #[classattr]
    const TITLE_END: u32 = 131121;
    /// Start of section - semantic document section (131122)
    #[classattr]
    const SECTION: u32 = 131122;
    /// End of section (131123)
    #[classattr]
    const SECTION_END: u32 = 131123;
    /// Start of summary - condensed content summary (131124)
    #[classattr]
    const SUMMARY: u32 = 131124;
    /// End of summary (131125)
    #[classattr]
    const SUMMARY_END: u32 = 131125;

}

/// Register all agent token classes with the Python module.
pub fn register_agent_tokens(m: &Bound<'_, PyModule>) -> PyResult<()> {
    m.add_class::<PyCL100KAgentTokens>()?;
    m.add_class::<PyO200KAgentTokens>()?;
    m.add_class::<PyLlama3AgentTokens>()?;
    m.add_class::<PyDeepSeekV3AgentTokens>()?;
    m.add_class::<PyMistralV1AgentTokens>()?;
    m.add_class::<PyMistralV2AgentTokens>()?;
    m.add_class::<PyMistralV3AgentTokens>()?;
    Ok(())
}