ruvector-attention 2.1.0

Attention mechanisms for ruvector - geometric, graph, and sparse attention
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

//! Pre-configured attention presets for common use cases.

use crate::sdk::builder::AttentionBuilder;

#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AttentionPreset {
    Bert,
    Gpt,
    Longformer,
    Performer,
    FlashOptimized,
    SwitchTransformer,
    HyperbolicTree,
    T5,
    ViT,
    SparseTransformer,
}

impl AttentionPreset {
    pub fn builder(self, dim: usize) -> AttentionBuilder {
        match self {
            AttentionPreset::Bert => AttentionBuilder::new(dim).multi_head(12).dropout(0.1),
            AttentionPreset::Gpt => AttentionBuilder::new(dim)
                .multi_head(12)
                .causal(true)
                .dropout(0.1),
            _ => AttentionBuilder::new(dim),
        }
    }
}

pub fn for_sequences(dim: usize, _max_len: usize) -> AttentionBuilder {
    AttentionBuilder::new(dim)
}

pub fn for_graphs(dim: usize, _hierarchical: bool) -> AttentionBuilder {
    AttentionBuilder::new(dim)
}

pub fn for_large_scale(dim: usize) -> AttentionBuilder {
    AttentionBuilder::new(dim).flash(128)
}