1use serde::{Deserialize, Serialize};
2
3#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
4pub enum ModelKind {
5 Embedding,
6 Reranker,
7}
8
9#[derive(Debug, Clone)]
11pub struct ModelFile {
12 pub name: &'static str,
13 pub sha256: &'static str,
14 pub size: u64,
15}
16
17#[derive(Debug, Clone)]
18pub struct ModelSpec {
19 pub id: &'static str,
20 pub name: &'static str,
21 pub kind: ModelKind,
22 pub repo: &'static str,
25 pub revision: &'static str,
27 pub files: &'static [ModelFile],
28 pub license: &'static str,
29 pub languages: &'static str,
30 pub dims: usize,
32 pub max_tokens: usize,
33 pub query_prefix: &'static str,
36 pub document_prefix: &'static str,
37}
38
39impl ModelSpec {
40 pub fn size_bytes(&self) -> u64 {
41 self.files.iter().map(|f| f.size).sum()
42 }
43
44 pub fn size_mb(&self) -> u64 {
45 self.size_bytes() / 1_000_000
46 }
47
48 pub fn file(&self, name: &str) -> Option<&ModelFile> {
49 self.files.iter().find(|f| f.name == name)
50 }
51
52 pub fn download_urls(&self, file_name: &str) -> Vec<String> {
61 vec![
62 format!(
63 "https://github.com/arrowassassin/paperscope/releases/download/v{}/{}-{}",
64 env!("CARGO_PKG_VERSION"),
65 self.id,
66 file_name
67 ),
68 format!(
69 "https://huggingface.co/{}/resolve/{}/{}",
70 self.repo, self.revision, file_name
71 ),
72 ]
73 }
74}
75
76pub const DEFAULT_EMBEDDING: &str = "bge-small-en-v1.5";
78pub const MULTILINGUAL_EMBEDDING: &str = "multilingual-e5-small";
80pub const OPTIONAL_RERANKER: &str = "bge-reranker-base";
82
83pub const EMBEDDING_DIMS: usize = 384;
86
87const BGE_SMALL_FILES: &[ModelFile] = &[
88 ModelFile {
89 name: "model.safetensors",
90 sha256: "3c9f31665447c8911517620762200d2245a2518d6e7208acc78cd9db317e21ad",
91 size: 133_466_304,
92 },
93 ModelFile {
94 name: "config.json",
95 sha256: "094f8e891b932f2000c92cfc663bac4c62069f5d8af5b5278c4306aef3084750",
96 size: 743,
97 },
98 ModelFile {
99 name: "tokenizer.json",
100 sha256: "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66",
101 size: 711_396,
102 },
103];
104
105const E5_SMALL_FILES: &[ModelFile] = &[
106 ModelFile {
107 name: "model.safetensors",
108 sha256: "1a55775f53449dac10a2bcbc312469fac40b96d53198c407081a831f81c98477",
109 size: 470_641_600,
110 },
111 ModelFile {
112 name: "config.json",
113 sha256: "69137736cab8b8903a07fe8afaafdda25aac55415a12a55d1bffa9f581abf959",
114 size: 655,
115 },
116 ModelFile {
117 name: "tokenizer.json",
118 sha256: "0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39",
119 size: 17_082_730,
120 },
121];
122
123const BGE_RERANKER_FILES: &[ModelFile] = &[
124 ModelFile {
125 name: "model.safetensors",
126 sha256: "ced967c45fd1902eb92716c9ceeca7c95a936770ea9db611f5a841b926e33fbd",
127 size: 1_112_206_140,
128 },
129 ModelFile {
130 name: "config.json",
131 sha256: "289adf7ada1eb6b4afa7589a48a032d45a076cf2e46dcdb3b4cabc33be14f708",
132 size: 799,
133 },
134 ModelFile {
135 name: "tokenizer.json",
136 sha256: "9eb652ac4e40cc093272bbbe0f55d521cf67570060227109b5cdc20945a4489e",
137 size: 17_098_107,
138 },
139];
140
141pub fn all_specs() -> Vec<ModelSpec> {
142 vec![
143 ModelSpec {
144 id: DEFAULT_EMBEDDING,
145 name: "BGE small English v1.5 (default, 384 dims)",
146 kind: ModelKind::Embedding,
147 repo: "BAAI/bge-small-en-v1.5",
148 revision: "5c38ec7c405ec4b44b94cc5a9bb96e735b38267a",
149 files: BGE_SMALL_FILES,
150 license: "MIT",
151 languages: "English",
152 dims: 384,
153 max_tokens: 512,
154 query_prefix: "Represent this sentence for searching relevant passages: ",
155 document_prefix: "",
156 },
157 ModelSpec {
158 id: MULTILINGUAL_EMBEDDING,
159 name: "Multilingual E5 small (opt-in, 384 dims)",
160 kind: ModelKind::Embedding,
161 repo: "intfloat/multilingual-e5-small",
162 revision: "614241f622f53c4eeff9890bdc4f31cfecc418b3",
163 files: E5_SMALL_FILES,
164 license: "MIT",
165 languages: "100+ languages",
166 dims: 384,
167 max_tokens: 512,
168 query_prefix: "query: ",
169 document_prefix: "passage: ",
170 },
171 ModelSpec {
172 id: OPTIONAL_RERANKER,
173 name: "BGE reranker base (opt-in precision)",
174 kind: ModelKind::Reranker,
175 repo: "BAAI/bge-reranker-base",
176 revision: "2cfc18c9415c912f9d8155881c133215df768a70",
177 files: BGE_RERANKER_FILES,
178 license: "MIT",
179 languages: "Multilingual",
180 dims: 0,
181 max_tokens: 512,
182 query_prefix: "",
183 document_prefix: "",
184 },
185 ]
186}
187
188pub fn spec(model_id: &str) -> Option<ModelSpec> {
189 all_specs().into_iter().find(|s| s.id == model_id)
190}
191
192#[cfg(test)]
193mod tests {
194 use super::*;
195
196 #[test]
197 fn download_urls_prefer_github_releases_with_dist_asset_names() {
198 let spec = spec(DEFAULT_EMBEDDING).unwrap();
199 let urls = spec.download_urls("model.safetensors");
200 assert_eq!(urls.len(), 2);
201 assert_eq!(
204 urls[0],
205 format!(
206 "https://github.com/arrowassassin/paperscope/releases/download/v{}/bge-small-en-v1.5-model.safetensors",
207 env!("CARGO_PKG_VERSION")
208 )
209 );
210 assert!(urls[1].starts_with("https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/"));
212 assert!(urls[1].ends_with("/model.safetensors"));
213 }
214}