rust_bert/models/deberta/
mod.rs

1//! # DeBERTa :Decoding-enhanced BERT with Disentangled Attention (He et al.)
2//!
3//! Implementation of the DeBERTa language model ([DeBERTa :Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) He, Liu ,Gao, Chen, 2021).
4//! The base model is implemented in the `deberta_model::DebertaModel` struct. Several language model heads have also been implemented, including:
5//! - Question answering: `deberta_model::DebertaForQuestionAnswering`
6//! - Sequence classification: `deberta_model::DebertaForSequenceClassification`
7//! - Token classification (e.g. NER, POS tagging): `deberta_model::DebertaForTokenClassification`.
8//!
9//! # Model set-up and pre-trained weights loading
10//!
11//! All models expect the following resources:
12//! - Configuration file expected to have a structure following the [Transformers library](https://github.com/huggingface/transformers)
13//! - Model weights are expected to have a structure and parameter names following the [Transformers library](https://github.com/huggingface/transformers). A conversion using the Python utility scripts is required to convert the `.bin` weights to the `.ot` format.
14//! - `DebertaTokenizer` using a `vocab.json` vocabulary and `merges.txt` merges file
15//!
16//! Pretrained models for a number of language pairs are available and can be downloaded using RemoteResources.
17//!
18//! ```no_run
19//! # fn main() -> anyhow::Result<()> {
20//! #
21//! use tch::{nn, Device};
22//! # use std::path::PathBuf;
23//! use rust_bert::deberta::{
24//!     DebertaConfig, DebertaConfigResources, DebertaForSequenceClassification,
25//!     DebertaMergesResources, DebertaModelResources, DebertaVocabResources,
26//! };
27//! use rust_bert::resources::{RemoteResource, ResourceProvider};
28//! use rust_bert::Config;
29//! use rust_tokenizers::tokenizer::DeBERTaTokenizer;
30//!
31//! let config_resource =
32//!     RemoteResource::from_pretrained(DebertaConfigResources::DEBERTA_BASE_MNLI);
33//! let vocab_resource = RemoteResource::from_pretrained(DebertaVocabResources::DEBERTA_BASE_MNLI);
34//! let merges_resource =
35//!     RemoteResource::from_pretrained(DebertaMergesResources::DEBERTA_BASE_MNLI);
36//! let weights_resource =
37//!     RemoteResource::from_pretrained(DebertaModelResources::DEBERTA_BASE_MNLI);
38//! let config_path = config_resource.get_local_path()?;
39//! let vocab_path = vocab_resource.get_local_path()?;
40//! let merges_path = merges_resource.get_local_path()?;
41//! let weights_path = weights_resource.get_local_path()?;
42//! let device = Device::cuda_if_available();
43//! let mut vs = nn::VarStore::new(device);
44//! let tokenizer = DeBERTaTokenizer::from_file(
45//!     vocab_path.to_str().unwrap(),
46//!     merges_path.to_str().unwrap(),
47//!     true,
48//! )?;
49//! let config = DebertaConfig::from_file(config_path);
50//! let deberta_model = DebertaForSequenceClassification::new(&vs.root(), &config);
51//! vs.load(weights_path)?;
52//!
53//! # Ok(())
54//! # }
55//! ```
56
57mod attention;
58mod deberta_model;
59mod embeddings;
60mod encoder;
61
62pub use deberta_model::{
63    DebertaConfig, DebertaConfigResources, DebertaForMaskedLM, DebertaForQuestionAnswering,
64    DebertaForSequenceClassification, DebertaForTokenClassification, DebertaMaskedLMOutput,
65    DebertaMergesResources, DebertaModel, DebertaModelResources, DebertaQuestionAnsweringOutput,
66    DebertaSequenceClassificationOutput, DebertaTokenClassificationOutput, DebertaVocabResources,
67};
68
69pub(crate) use deberta_model::{
70    deserialize_attention_type, x_softmax, BaseDebertaLayerNorm, ContextPooler,
71    DebertaLMPredictionHead, DebertaModelOutput, PositionAttentionType, PositionAttentionTypes,
72};
73
74pub(crate) use attention::{DebertaDisentangledSelfAttention, DisentangledSelfAttention};
75pub(crate) use embeddings::BaseDebertaEmbeddings;
76pub(crate) use encoder::{BaseDebertaLayer, DebertaEncoderOutput};
rust_bert/models/deberta/mod.rs

rust_bert/models/deberta/
mod.rs
