Skip to main content

paper/core/models/
specs.rs

1use serde::{Deserialize, Serialize};
2
3#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
4pub enum ModelKind {
5    Embedding,
6    Reranker,
7}
8
9/// One downloadable artifact, pinned by content hash.
10#[derive(Debug, Clone)]
11pub struct ModelFile {
12    pub name: &'static str,
13    pub sha256: &'static str,
14    pub size: u64,
15}
16
17#[derive(Debug, Clone)]
18pub struct ModelSpec {
19    pub id: &'static str,
20    pub name: &'static str,
21    pub kind: ModelKind,
22    /// First-party Hugging Face repository. Only ungated repositories are used:
23    /// installing Paper never requires an account or a token.
24    pub repo: &'static str,
25    /// Pinned commit, so a re-upload upstream cannot change what we run.
26    pub revision: &'static str,
27    pub files: &'static [ModelFile],
28    pub license: &'static str,
29    pub languages: &'static str,
30    /// Embedding width. `0` for rerankers, which produce a scalar.
31    pub dims: usize,
32    pub max_tokens: usize,
33    /// Retrieval instruction prefixes. BGE wants one on queries only; E5 wants
34    /// them on both sides. Getting these wrong measurably hurts recall.
35    pub query_prefix: &'static str,
36    pub document_prefix: &'static str,
37}
38
39impl ModelSpec {
40    pub fn size_bytes(&self) -> u64 {
41        self.files.iter().map(|f| f.size).sum()
42    }
43
44    pub fn size_mb(&self) -> u64 {
45        self.size_bytes() / 1_000_000
46    }
47
48    pub fn file(&self, name: &str) -> Option<&ModelFile> {
49        self.files.iter().find(|f| f.name == name)
50    }
51
52    /// Candidate download URLs for one file, in preference order.
53    ///
54    /// The primary source is this crate's own GitHub Release, which carries
55    /// the exact pinned bytes as `<model-id>-<file>` assets (staged by
56    /// `scripts/fetch-models.sh` during release CI). Hugging Face is the
57    /// fallback for development builds whose `v<version>` tag has no release
58    /// yet. Both sources are verified against the same SHA256 pin, so either
59    /// is safe to consume.
60    pub fn download_urls(&self, file_name: &str) -> Vec<String> {
61        vec![
62            format!(
63                "https://github.com/arrowassassin/paperscope/releases/download/v{}/{}-{}",
64                env!("CARGO_PKG_VERSION"),
65                self.id,
66                file_name
67            ),
68            format!(
69                "https://huggingface.co/{}/resolve/{}/{}",
70                self.repo, self.revision, file_name
71            ),
72        ]
73    }
74}
75
76/// Default: small, fast, English. 134 MB total.
77pub const DEFAULT_EMBEDDING: &str = "bge-small-en-v1.5";
78/// Opt-in for multilingual households. 488 MB total.
79pub const MULTILINGUAL_EMBEDDING: &str = "multilingual-e5-small";
80/// Opt-in cross-encoder for precision. 1.13 GB total, multilingual.
81pub const OPTIONAL_RERANKER: &str = "bge-reranker-base";
82
83/// Width of the vectors in `chunks_vec`. Both supported embedders are 384-dim,
84/// so one index can be rebuilt from either without a schema change.
85pub const EMBEDDING_DIMS: usize = 384;
86
87const BGE_SMALL_FILES: &[ModelFile] = &[
88    ModelFile {
89        name: "model.safetensors",
90        sha256: "3c9f31665447c8911517620762200d2245a2518d6e7208acc78cd9db317e21ad",
91        size: 133_466_304,
92    },
93    ModelFile {
94        name: "config.json",
95        sha256: "094f8e891b932f2000c92cfc663bac4c62069f5d8af5b5278c4306aef3084750",
96        size: 743,
97    },
98    ModelFile {
99        name: "tokenizer.json",
100        sha256: "d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66",
101        size: 711_396,
102    },
103];
104
105const E5_SMALL_FILES: &[ModelFile] = &[
106    ModelFile {
107        name: "model.safetensors",
108        sha256: "1a55775f53449dac10a2bcbc312469fac40b96d53198c407081a831f81c98477",
109        size: 470_641_600,
110    },
111    ModelFile {
112        name: "config.json",
113        sha256: "69137736cab8b8903a07fe8afaafdda25aac55415a12a55d1bffa9f581abf959",
114        size: 655,
115    },
116    ModelFile {
117        name: "tokenizer.json",
118        sha256: "0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39",
119        size: 17_082_730,
120    },
121];
122
123const BGE_RERANKER_FILES: &[ModelFile] = &[
124    ModelFile {
125        name: "model.safetensors",
126        sha256: "ced967c45fd1902eb92716c9ceeca7c95a936770ea9db611f5a841b926e33fbd",
127        size: 1_112_206_140,
128    },
129    ModelFile {
130        name: "config.json",
131        sha256: "289adf7ada1eb6b4afa7589a48a032d45a076cf2e46dcdb3b4cabc33be14f708",
132        size: 799,
133    },
134    ModelFile {
135        name: "tokenizer.json",
136        sha256: "9eb652ac4e40cc093272bbbe0f55d521cf67570060227109b5cdc20945a4489e",
137        size: 17_098_107,
138    },
139];
140
141pub fn all_specs() -> Vec<ModelSpec> {
142    vec![
143        ModelSpec {
144            id: DEFAULT_EMBEDDING,
145            name: "BGE small English v1.5 (default, 384 dims)",
146            kind: ModelKind::Embedding,
147            repo: "BAAI/bge-small-en-v1.5",
148            revision: "5c38ec7c405ec4b44b94cc5a9bb96e735b38267a",
149            files: BGE_SMALL_FILES,
150            license: "MIT",
151            languages: "English",
152            dims: 384,
153            max_tokens: 512,
154            query_prefix: "Represent this sentence for searching relevant passages: ",
155            document_prefix: "",
156        },
157        ModelSpec {
158            id: MULTILINGUAL_EMBEDDING,
159            name: "Multilingual E5 small (opt-in, 384 dims)",
160            kind: ModelKind::Embedding,
161            repo: "intfloat/multilingual-e5-small",
162            revision: "614241f622f53c4eeff9890bdc4f31cfecc418b3",
163            files: E5_SMALL_FILES,
164            license: "MIT",
165            languages: "100+ languages",
166            dims: 384,
167            max_tokens: 512,
168            query_prefix: "query: ",
169            document_prefix: "passage: ",
170        },
171        ModelSpec {
172            id: OPTIONAL_RERANKER,
173            name: "BGE reranker base (opt-in precision)",
174            kind: ModelKind::Reranker,
175            repo: "BAAI/bge-reranker-base",
176            revision: "2cfc18c9415c912f9d8155881c133215df768a70",
177            files: BGE_RERANKER_FILES,
178            license: "MIT",
179            languages: "Multilingual",
180            dims: 0,
181            max_tokens: 512,
182            query_prefix: "",
183            document_prefix: "",
184        },
185    ]
186}
187
188pub fn spec(model_id: &str) -> Option<ModelSpec> {
189    all_specs().into_iter().find(|s| s.id == model_id)
190}
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195
196    #[test]
197    fn download_urls_prefer_github_releases_with_dist_asset_names() {
198        let spec = spec(DEFAULT_EMBEDDING).unwrap();
199        let urls = spec.download_urls("model.safetensors");
200        assert_eq!(urls.len(), 2);
201        // Must match the staged asset names in dist-workspace.toml:
202        // `models/release/<model-id>-<file>`.
203        assert_eq!(
204            urls[0],
205            format!(
206                "https://github.com/arrowassassin/paperscope/releases/download/v{}/bge-small-en-v1.5-model.safetensors",
207                env!("CARGO_PKG_VERSION")
208            )
209        );
210        // Hugging Face stays the fallback, pinned to the same revision.
211        assert!(urls[1].starts_with("https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/"));
212        assert!(urls[1].ends_with("/model.safetensors"));
213    }
214}