burn_dragon_language 0.4.0

Language modeling components for burn_dragon
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

#![recursion_limit = "256"]

pub mod config;
pub mod generation;
pub mod inference;
pub mod tokenizer;

#[cfg(feature = "train")]
pub mod dataset;
#[cfg(feature = "train")]
pub mod train;

pub use config::{ContextStrategyConfig, GenerationConfig, ModelOverrides};
#[cfg(feature = "train")]
pub use config::{
    DatasetConfig, DatasetSourceConfig, HuggingFaceDatasetConfig, HuggingFaceRecordFormat,
    TrainingConfig, TrainingHyperparameters, load_training_config,
};
pub use generation::{
    ContextStrategy, GenerationSettings, generate_text, generate_tokens, prefill_state,
    resolve_context_strategy, sample_next_token,
};
pub use inference::build_model_config;
pub use tokenizer::char_vocab::CharVocab;