List of all items
Structs
- config::IngestBuilder
- config::IngestConfig
- config::PreprocessorConfig
- config::TrainerBuilder
- config::TrainerConfig
- corpus::BinaryChunkStream
- corpus::JsonlSpec
- corpus::JsonlStream
- metrics::IterationMetrics
- metrics::TrainingMetrics
- model::BinaryTokenizer
- model::BpeModel
- preprocess::PreprocessorRunner
- trainer::SequenceStream
- trainer::Trainer
- trainer::TrainerArtifacts
Enums
- bytes::ByteEncoding
- config::PreprocessorKind
- error::BbpeError
- metrics::StopReason
- preprocess::PreprocessedSequences
Functions
- bytes::bytes_to_latin1
- bytes::bytes_to_string
- bytes::contains_ascii_letter
- bytes::ends_with_ascii_whitespace
- bytes::is_all_ascii_whitespace
- bytes::is_allowed_length
- bytes::is_ascii_whitespace
- bytes::latin1_to_bytes
- bytes::legacy_bytes_to_latin1
- bytes::legacy_latin1_to_bytes
- bytes::starts_with_ascii_whitespace
- bytes::string_to_bytes
- corpus::collect_paths
- corpus::load_binary_corpus
- corpus::load_jsonl_corpus
- corpus::stream_binary_corpus
- corpus::stream_jsonl_corpus
- metrics::sample_rss_kb
- preprocess::apply_preprocessor
- serialization::huggingface::as_tokenizer
- serialization::huggingface::load_tokenizer
- serialization::huggingface::save_huggingface_tokenizer
- serialization::huggingface::tokenizer_json
- special_tokens::dedup_in_place
- special_tokens::leading_tokens
- special_tokens::reasoning_tokens
- special_tokens::trailing_tokens