Struct RetNetConfig

Source

pub struct RetNetConfig {Show 31 fields
    pub vocab_size: usize,
    pub hidden_size: usize,
    pub num_hidden_layers: usize,
    pub num_heads: usize,
    pub intermediate_size: usize,
    pub hidden_act: String,
    pub hidden_dropout_prob: f32,
    pub attention_dropout_prob: f32,
    pub max_position_embeddings: usize,
    pub initializer_range: f32,
    pub layer_norm_eps: f32,
    pub pad_token_id: u32,
    pub bos_token_id: u32,
    pub eos_token_id: u32,
    pub use_bias: bool,
    pub use_glu: bool,
    pub use_norm_bias: bool,
    pub deepnorm: bool,
    pub dropout_module: String,
    pub activation_dropout: f32,
    pub attention_dropout: f32,
    pub retention_heads: usize,
    pub value_factor: f32,
    pub gate_fn: String,
    pub tensor_parallel_degree: usize,
    pub sequence_parallel: bool,
    pub fuse_norm: bool,
    pub no_output_layer: bool,
    pub layernorm_embedding: bool,
    pub chunking: bool,
    pub chunk_size: usize,
}

Expand description

RetNet model configuration Reference: “Retentive Network: A Successor to Transformer for Large Language Models”

Fields§

§vocab_size: usize§hidden_size: usize§num_hidden_layers: usize§num_heads: usize§intermediate_size: usize§hidden_act: String§hidden_dropout_prob: f32§attention_dropout_prob: f32§max_position_embeddings: usize§initializer_range: f32§layer_norm_eps: f32§pad_token_id: u32§bos_token_id: u32§eos_token_id: u32§use_bias: bool§use_glu: bool§use_norm_bias: bool§deepnorm: bool§dropout_module: String§activation_dropout: f32§attention_dropout: f32§retention_heads: usize§value_factor: f32§gate_fn: String§tensor_parallel_degree: usize§sequence_parallel: bool§fuse_norm: bool§no_output_layer: bool§layernorm_embedding: bool§chunking: bool§chunk_size: usize

Struct RetNetConfig Copy item path

Fields§

Implementations§

impl RetNetConfig

pub fn retnet_small() -> Self

pub fn retnet_medium() -> Self

pub fn retnet_large() -> Self

pub fn retnet_xl() -> Self

pub fn retnet_long() -> Self

pub fn head_dim(&self) -> usize

pub fn retention_head_dim(&self) -> usize

pub fn retention_dim(&self) -> usize

pub fn uses_chunking(&self) -> bool

pub fn memory_advantage(&self) -> f32

pub fn supports_long_sequences(&self) -> bool

pub fn deepnorm_alpha(&self) -> f32

pub fn deepnorm_beta(&self) -> f32

Trait Implementations§

impl Clone for RetNetConfig

fn clone(&self) -> RetNetConfig

fn clone_from(&mut self, source: &Self)

impl Config for RetNetConfig

fn validate(&self) -> Result<()>

fn architecture(&self) -> &'static str

impl Debug for RetNetConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

impl Default for RetNetConfig

fn default() -> Self

impl<'de> Deserialize<'de> for RetNetConfig

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where __D: Deserializer<'de>,

impl Serialize for RetNetConfig

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where __S: Serializer,

Auto Trait Implementations§

impl Freeze for RetNetConfig

impl RefUnwindSafe for RetNetConfig

impl Send for RetNetConfig

impl Sync for RetNetConfig

impl Unpin for RetNetConfig

impl UnsafeUnpin for RetNetConfig

impl UnwindSafe for RetNetConfig

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> CloneToUninit for Twhere T: Clone,

unsafe fn clone_to_uninit(&self, dest: *mut u8)

impl<T> ConfigSerializable for Twhere T: Serialize + for<'de> Deserialize<'de>,

fn to_json(&self) -> Result<String, TrustformersError>

fn from_json(json: &str) -> Result<T, TrustformersError>

fn save_to_file(&self, path: &Path) -> Result<(), TrustformersError>

fn load_from_file(path: &Path) -> Result<Self, TrustformersError>where Self: Sized,

impl<T> DynConfig for Twhere T: Config + 'static,

fn validate(&self) -> Result<(), TrustformersError>

fn architecture(&self) -> &'static str

fn as_any(&self) -> &(dyn Any + 'static)

impl<T> From<T> for T

fn from(t: T) -> T

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self>

fn in_current_span(self) -> Instrumented<Self>

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self>

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>where F: FnOnce(&Self) -> bool,

impl<T> Pointable for T

const ALIGN: usize

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T> PolicyExt for Twhere T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>where T: Policy<B, E>, P: Policy<B, E>,

impl<T> Same for T

type Output = T

Struct RetNetConfig

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where __D: Deserializer<'de>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where S: Serializer,

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T> CloneToUninit for T
where T: Clone,

impl<T> ConfigSerializable for T
where T: Serialize + for<'de> Deserialize<'de>,

fn load_from_file(path: &Path) -> Result<Self, TrustformersError>
where Self: Sized,

impl<T> DynConfig for T
where T: Config + 'static,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

impl<T> PolicyExt for T
where T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

impl<T> ToOwned for T
where T: Clone,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

impl<T> DeserializeOwned for T
where T: for<'de> Deserialize<'de>,