Struct LLaDA2MoeConfig 

pub struct LLaDA2MoeConfig {

    pub vocab_size: usize,
    pub hidden_size: usize,
    pub intermediate_size: Option<usize>,
    pub num_hidden_layers: usize,
    pub num_attention_heads: usize,
    pub num_key_value_heads: usize,
    pub head_dim: Option<usize>,
    pub num_experts: usize,
    pub num_experts_per_tok: usize,
    pub num_shared_experts: Option<usize>,
    pub moe_intermediate_size: Option<usize>,
    pub n_group: usize,
    pub topk_group: usize,
    pub routed_scaling_factor: f32,
    pub first_k_dense_replace: usize,
    pub max_position_embeddings: usize,
    pub rope_theta: f64,
    pub rms_norm_eps: f64,
    pub partial_rotary_factor: f32,
    pub use_qk_norm: bool,
    pub use_qkv_bias: bool,
    pub use_bias: bool,
    pub hidden_act: String,
    pub attention_dropout: f64,
    pub embedding_dropout: f64,
    pub output_dropout: f64,
    pub tie_word_embeddings: bool,
    pub norm_topk_prob: bool,
    pub moe_router_enable_expert_bias: bool,
    pub pad_token_id: u32,
    pub mask_token_id: u32,
    pub eos_token_id: u32,
}

Fields

vocab_size: usize

hidden_size: usize

intermediate_size: Option<usize>

num_hidden_layers: usize

num_attention_heads: usize

num_key_value_heads: usize

head_dim: Option<usize>

num_experts: usize

num_experts_per_tok: usize

num_shared_experts: Option<usize>

moe_intermediate_size: Option<usize>

n_group: usize

topk_group: usize

routed_scaling_factor: f32

first_k_dense_replace: usize

max_position_embeddings: usize

rope_theta: f64

rms_norm_eps: f64

partial_rotary_factor: f32

use_qk_norm: bool

use_qkv_bias: bool

use_bias: bool

hidden_act: String

attention_dropout: f64

embedding_dropout: f64

output_dropout: f64

tie_word_embeddings: bool

norm_topk_prob: bool

moe_router_enable_expert_bias: bool

pad_token_id: u32

mask_token_id: u32

eos_token_id: u32

Implementations

impl LLaDA2MoeConfig

pub fn from_json_str(s: &str) -> Result<LLaDA2MoeConfig, Error>

pub fn from_file(path: &Path) -> Result<LLaDA2MoeConfig, Error>

pub fn from_tide_repo() -> Result<LLaDA2MoeConfig, Error>

pub fn head_dim(&self) -> usize

pub fn intermediate_size(&self) -> usize

pub fn expert_ffn_dim(&self) -> usize

pub fn num_kv_heads(&self) -> usize

pub fn kv_group_size(&self) -> usize

pub fn rope_dim(&self) -> usize

pub fn is_moe_layer(&self, layer: usize) -> bool

pub fn num_sparse_moe_layers(&self) -> usize

pub fn expert_param_bytes_f32(&self) -> usize

Trait Implementations

impl Clone for LLaDA2MoeConfig

fn clone(&self) -> LLaDA2MoeConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for LLaDA2MoeConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for LLaDA2MoeConfig

fn deserialize<__D>( __deserializer: __D, ) -> Result<LLaDA2MoeConfig, <__D as Deserializer<'de>>::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.