gliner2 0.1.2

Rust implementation of GLiNER2 with compatibility for upstream weights and Python training output.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

use anyhow::Result;
use hf_hub::{Repo, RepoType, api::sync::Api};
use serde::{Deserialize, Serialize};
use std::path::PathBuf;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExtractorConfig {
    pub model_name: String,
    pub max_width: usize,
    pub counting_layer: String,
    pub token_pooling: String,
    pub max_len: Option<usize>,
}

impl Default for ExtractorConfig {
    fn default() -> Self {
        Self {
            model_name: "microsoft/deberta-v3-base".to_string(),
            max_width: 8,
            counting_layer: "count_lstm".to_string(),
            token_pooling: "first".to_string(),
            max_len: None,
        }
    }
}

pub struct ModelFiles {
    pub config: PathBuf,
    pub encoder_config: PathBuf,
    pub tokenizer: PathBuf,
    pub weights: PathBuf,
}

pub fn download_model(repo_id: &str) -> Result<ModelFiles> {
    let api = Api::new()?;
    let repo = api.repo(Repo::new(repo_id.to_string(), RepoType::Model));

    let config = repo.get("config.json")?;
    let encoder_config = repo.get("encoder_config/config.json")?;
    let tokenizer = repo.get("tokenizer.json")?;
    let weights = repo.get("model.safetensors")?;

    Ok(ModelFiles {
        config,
        encoder_config,
        tokenizer,
        weights,
    })
}