rustam 0.1.0

Full Persian NLP pipeline for Rust — normalization, tokenization, stemming, lemmatization, conjugation, spell correction, and more
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

//! Hugging Face Hub model downloader.
//!
//! Downloads pre-trained Persian NLP models from the
//! [roshan-research](https://huggingface.co/roshan-research) HF Hub organisation
//! and returns their local cache path.
//!
//! Enabled by the `hf-hub` Cargo feature.
//!
//! # Example
//!
//! ```ignore
//! use rustam::download::{download, Model};
//!
//! let pos_path = download(Model::PosTagger).unwrap();
//! let ft_path  = download(Model::FastText).unwrap();
//! ```

#[cfg(feature = "hf-hub")]
use std::path::PathBuf;

#[cfg(feature = "hf-hub")]
use crate::error::{Error, Result};

/// Pre-trained model identifiers understood by [`download`].
pub enum Model {
    /// CRF POS tagger model (`pos_tagger.model`)
    PosTagger,
    /// CRF chunker model (`chunker.model`)
    Chunker,
    /// CRF NER model (`ner.model`)
    Ner,
    /// Arc-eager dependency parser model (`dep.model`)
    DepParser,
    /// FastText skip-gram 300-d Persian embeddings (`fasttext_skipgram_300.bin`)
    FastText,
    /// Word2Vec Persian embeddings (`word2vec.bin`)
    Word2Vec,
}

/// Downloads `model` from HF Hub and returns its local cache path.
///
/// Files are cached in the standard HF Hub cache directory and only
/// re-downloaded when the remote copy changes.
///
/// Requires the `hf-hub` Cargo feature.
#[cfg(feature = "hf-hub")]
pub fn download(model: Model) -> Result<PathBuf> {
    use hf_hub::api::sync::Api;

    let (repo_id, filename): (&str, &str) = match model {
        Model::PosTagger => ("amirhosseinghanipour/rustam-postagger",        "pos_tagger.model"),
        Model::Chunker   => ("amirhosseinghanipour/rustam-chunker",          "chunker.model"),
        Model::Ner       => ("amirhosseinghanipour/rustam-ner",              "ner.model"),
        Model::DepParser => ("amirhosseinghanipour/rustam-dependency-parser", "dep.model"),
        Model::FastText  => ("amirhosseinghanipour/rustam-word-embedding",    "fasttext_skipgram_300.bin"),
        Model::Word2Vec  => ("amirhosseinghanipour/rustam-word-embedding",    "word2vec.bin"),
    };

    let api = Api::new().map_err(|e| Error::Parse(e.to_string()))?;
    api.model(repo_id.to_string())
        .get(filename)
        .map_err(|e| Error::Parse(format!("HF Hub download failed: {e}")))
}