rlx-text 0.2.14

RLX text — tokenizer wrappers, chat templates, sampling. Public surface for downstream LM apps.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

// RLX — versatile ML compiler + runtime.
// Copyright (C) 2026 Eugene Hauptmann, Nataliya Kosmyna.
// SPDX-License-Identifier: MIT OR Apache-2.0

//! RLX text: tokenizer + chat template + sampling helpers.
//!
//! Promoted from `rlx-models/crates/rlx-cli` so that downstream LM apps
//! (server, web playground, training tools) have one published crate to
//! depend on without taking the CLI helper layer.

pub mod chat;
pub mod detokenize;
pub mod sampling;
pub mod tokenizer;
pub mod tool_parse;

pub use chat::{ChatMessage, ChatTemplate, ChatTemplateSource, auto_chat_template};
pub use detokenize::{StreamingDetokenizer, incremental_emit};
pub use rlx_runtime::SampleOpts;
pub use sampling::{argmax, sample_next};
pub use tokenizer::{TokenizerHandle, decode_ids, load_tokenizer};
pub use tool_parse::{ToolCall, ToolFormat};