Struct MiDataset 

pub struct MiDataset {
    pub tokens: Vec<usize>,
    pub tokenizer: MiTokenizer,
}

MI-tokenized dataset. Single-pass mutual information tokenizer (BA-37). ~500x faster than BPE on large corpora. φ-threshold significance gating.

Fields

tokens: Vec<usize>

tokenizer: MiTokenizer

Implementations

impl MiDataset

pub fn from_text(text: &str, target_vocab: usize) -> Self

pub fn from_jsonl(path: &Path, target_vocab: usize) -> Result<Self>

pub fn from_file(path: &Path, target_vocab: usize) -> Result<Self>

pub fn from_dir(path: &Path, target_vocab: usize) -> Result<Self>

Load from a directory of .txt files with MI tokenization.

pub fn from_path(path: &Path, target_vocab: usize) -> Result<Self>

Auto-detect format: .jsonl, .txt file, or directory of .txt files.

pub fn vocab_size(&self) -> usize

pub fn len(&self) -> usize

pub fn decode(&self, tokens: &[usize]) -> String