second-brain-core 0.13.4

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
use std::path::{Path, PathBuf};

use anyhow::{Context, Result};
use fastembed::{RerankerModel, TextRerank};
use ort::ep::{CPU, CUDA, ExecutionProvider, ExecutionProviderDispatch};
use ort::session::builder::GraphOptimizationLevel;
use ort::session::{Session, SessionInputValue};
use ort::value::Tensor;
use tokenizers::{AddedToken, PaddingParams, PaddingStrategy, Tokenizer, TruncationParams};

use super::rerank::RerankEp;
use crate::model_cache::{
    CachedModel, cached_snapshot, model_cache_dir, reranker_model, snapshots_dir,
};

/// Documents per ONNX `Run`. A batch is one tensor padded to its longest
/// pair, so mixing short chunk windows with 512-token memories made every
/// window cost a full-length pass; runs of this size over length-sorted
/// documents keep padding small while still batching the intra-op work.
const LENGTH_BUCKET: usize = 16;
/// Mirrors fastembed's `DEFAULT_MAX_LENGTH`; used only when the caller has
/// not overridden `SECOND_BRAIN_RERANK_MAX_LEN`.
const DEFAULT_MAX_LENGTH: usize = 512;

pub(crate) struct OnnxSessionConfig {
    pub(crate) intra_threads: usize,
    pub(crate) inter_threads: usize,
    pub(crate) opt_level: GraphOptimizationLevel,
    pub(crate) ep: RerankEp,
    pub(crate) max_length: Option<usize>,
}

pub(crate) fn default_opt_level() -> GraphOptimizationLevel {
    GraphOptimizationLevel::Level3
}

/// Cross-encoder session owned directly by this crate. `ort::session::Session`
/// is not `Sync`-callable (`Session::run` takes `&mut self`), so callers must
/// hold exclusive access for the duration of a rerank call, exactly like
/// `fastembed::TextRerank`.
pub(crate) struct OnnxReranker {
    tokenizer: Tokenizer,
    session: Session,
    need_token_type_ids: bool,
}

/// The cached snapshot holding every file `model` loads, in the Hugging Face
/// hub layout fastembed's own loader reads, so both backends share the files.
fn snapshot_dir(model: &CachedModel) -> Result<PathBuf> {
    let cache = model_cache_dir()?;
    cached_snapshot(&cache, model).with_context(|| {
        format!(
            "no complete cached snapshot for '{}' under {} (expected {} to already be on \
             disk; this backend does not download)",
            model.code,
            snapshots_dir(&cache, &model.code).display(),
            model.files.join(", ")
        )
    })
}

#[derive(Debug)]
struct ModelFiles {
    model_onnx: PathBuf,
    tokenizer_json: PathBuf,
    config_json: PathBuf,
    tokenizer_config_json: PathBuf,
    special_tokens_map_json: PathBuf,
}

fn resolve_model_files(model_name: RerankerModel) -> Result<ModelFiles> {
    resolve_model_files_with(model_name, download_snapshot)
}

/// `download` is injected so tests can verify the retry-after-download
/// orchestration without hitting the network; production always calls
/// this through `resolve_model_files`, which passes `download_snapshot`.
fn resolve_model_files_with(
    model_name: RerankerModel,
    download: impl FnOnce(RerankerModel) -> Result<()>,
) -> Result<ModelFiles> {
    let info = TextRerank::get_model_info(&model_name);
    let cached = reranker_model(&model_name);
    let snap = match snapshot_dir(&cached) {
        Ok(dir) => dir,
        Err(_) => {
            download(model_name)?;
            snapshot_dir(&cached)?
        }
    };
    Ok(ModelFiles {
        model_onnx: snap.join(&info.model_file),
        tokenizer_json: snap.join("tokenizer.json"),
        config_json: snap.join("config.json"),
        tokenizer_config_json: snap.join("tokenizer_config.json"),
        special_tokens_map_json: snap.join("special_tokens_map.json"),
    })
}

/// Downloads straight through `hf-hub` (not fastembed's `TextRerank::try_new`)
/// into the exact cache layout `snapshot_dir` reads from, so the caller
/// controls the token explicitly: `.with_token(None)` forces anonymous
/// access, since fastembed's own downloader exposes no token option and
/// otherwise defaults to whatever `~/.cache/huggingface/token` holds, which
/// belongs to unrelated tooling and has no reason to be valid for this.
/// These reranker models are all public and non-gated, so anonymous access
/// is correct, not just a workaround.
fn download_snapshot(model_name: RerankerModel) -> Result<()> {
    let model = reranker_model(&model_name);
    let api = hf_hub::api::sync::ApiBuilder::new()
        .with_cache_dir(model_cache_dir()?)
        .with_token(None)
        .with_progress(true)
        .build()
        .context("building hf-hub api client")?;
    let repo = api.model(model.code.clone());
    for filename in &model.files {
        repo.download(filename)
            .with_context(|| format!("downloading {filename} for {}", model.code))?;
    }
    Ok(())
}

fn read_json(path: &Path) -> Result<serde_json::Value> {
    let raw =
        std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
    serde_json::from_str(&raw).with_context(|| format!("parsing {} as json", path.display()))
}

/// Replicates fastembed's own tokenizer setup (`common::load_tokenizer`,
/// not part of fastembed's public API) field for field: `BatchLongest`
/// padding using the pad id/token taken from the model's own config files
/// rather than the tokenizers-crate default, and truncation capped to the
/// model's `model_max_length`. Gate 1 parity depends on this matching
/// fastembed byte-for-byte, since a different pad id, truncation bound, or
/// special-token set changes which tokens the model actually sees.
fn load_tokenizer(files: &ModelFiles, requested_max_length: Option<usize>) -> Result<Tokenizer> {
    let config = read_json(&files.config_json)?;
    let tokenizer_config = read_json(&files.tokenizer_config_json)?;
    let special_tokens_map = read_json(&files.special_tokens_map_json)?;

    let mut tokenizer = Tokenizer::from_file(&files.tokenizer_json)
        .map_err(|e| anyhow::anyhow!(e.to_string()))
        .with_context(|| format!("loading {}", files.tokenizer_json.display()))?;

    // model_max_length is stored as a float in some tokenizer_config.json
    // files (e.g. BGE's placeholder value doesn't fit in u64), so read it as
    // f64 first, matching fastembed's own parsing.
    let model_max_length = tokenizer_config["model_max_length"]
        .as_f64()
        .context("tokenizer_config.json missing model_max_length")?
        as usize;
    let max_length = requested_max_length
        .unwrap_or(DEFAULT_MAX_LENGTH)
        .min(model_max_length);

    let pad_id = config["pad_token_id"].as_u64().unwrap_or(0) as u32;
    let pad_token = tokenizer_config["pad_token"]
        .as_str()
        .context("tokenizer_config.json missing pad_token")?
        .to_string();

    tokenizer
        .with_padding(Some(PaddingParams {
            strategy: PaddingStrategy::BatchLongest,
            pad_token,
            pad_id,
            ..Default::default()
        }))
        .with_truncation(Some(TruncationParams {
            max_length,
            ..Default::default()
        }))
        .map_err(|e| anyhow::anyhow!(e.to_string()))
        .context("configuring tokenizer padding/truncation")?;

    if let serde_json::Value::Object(root) = special_tokens_map {
        for value in root.values() {
            if let Some(content) = value.as_str() {
                tokenizer.add_special_tokens(&[AddedToken {
                    content: content.into(),
                    special: true,
                    ..Default::default()
                }]);
            } else if let (
                Some(content),
                Some(single_word),
                Some(lstrip),
                Some(rstrip),
                Some(normalized),
            ) = (
                value["content"].as_str(),
                value["single_word"].as_bool(),
                value["lstrip"].as_bool(),
                value["rstrip"].as_bool(),
                value["normalized"].as_bool(),
            ) {
                tokenizer.add_special_tokens(&[AddedToken {
                    content: content.into(),
                    special: true,
                    single_word,
                    lstrip,
                    rstrip,
                    normalized,
                }]);
            }
        }
    }

    Ok(tokenizer)
}

fn builder_err(err: ort::Error<ort::session::builder::SessionBuilder>) -> anyhow::Error {
    anyhow::Error::msg(err.to_string())
}

/// Picks execution providers for the session. CUDA is only added when
/// `ort` itself reports the linked onnxruntime build supports it; there is
/// no GPU in this environment, so the CUDA branch here is exercised only by
/// the availability check (always false) and is otherwise untested end to
/// end.
fn execution_providers(ep: RerankEp) -> Vec<ExecutionProviderDispatch> {
    if ep != RerankEp::Cuda {
        return vec![CPU::default().build()];
    }
    let cuda = CUDA::default();
    let available = cuda.is_available().unwrap_or_else(|e| {
        tracing::warn!("checking CUDA execution provider availability failed: {e}");
        false
    });
    if available {
        vec![cuda.build(), CPU::default().build()]
    } else {
        tracing::warn!(
            "SECOND_BRAIN_RERANK_EP=cuda requested but onnxruntime in this build reports no \
             CUDA support; falling back to cpu"
        );
        vec![CPU::default().build()]
    }
}

fn build_session(model_path: &Path, config: &OnnxSessionConfig) -> Result<Session> {
    Session::builder()
        .context("creating onnx session builder")?
        .with_execution_providers(execution_providers(config.ep))
        .map_err(builder_err)
        .context("registering execution providers")?
        .with_optimization_level(config.opt_level)
        .map_err(builder_err)
        .context("setting graph optimization level")?
        .with_intra_threads(config.intra_threads)
        .map_err(builder_err)
        .context("setting intra-op thread count")?
        .with_inter_threads(config.inter_threads)
        .map_err(builder_err)
        .context("setting inter-op thread count")?
        .commit_from_file(model_path)
        .with_context(|| format!("committing onnx model {}", model_path.display()))
}

impl OnnxReranker {
    pub(crate) fn load(model_name: RerankerModel, config: &OnnxSessionConfig) -> Result<Self> {
        let files = resolve_model_files(model_name)?;
        let tokenizer = load_tokenizer(&files, config.max_length)?;
        let session = build_session(&files.model_onnx, config)?;
        let need_token_type_ids = session
            .inputs()
            .iter()
            .any(|input| input.name() == "token_type_ids");
        Ok(Self {
            tokenizer,
            session,
            need_token_type_ids,
        })
    }

    /// Cross-encoder forward pass over length-sorted runs of documents (see
    /// `LENGTH_BUCKET`); scores come back in the callers' document order.
    pub(crate) fn rerank_batch(&mut self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
        let mut scores = vec![0.0; documents.len()];
        for bucket in length_buckets(documents, LENGTH_BUCKET) {
            let docs: Vec<&str> = bucket.iter().map(|&i| documents[i]).collect();
            for (i, score) in bucket.into_iter().zip(self.run_batch(query, &docs)?) {
                scores[i] = score;
            }
        }
        Ok(scores)
    }

    /// One padded tensor for all of `batch`, one `Run`: batching is what lets
    /// intra-op threads work across the full matrix instead of paying
    /// per-call dispatch overhead once per document.
    fn run_batch(&mut self, query: &str, batch: &[&str]) -> Result<Vec<f32>> {
        let mut scores = Vec::with_capacity(batch.len());
        {
            let pairs: Vec<(&str, &str)> = batch.iter().map(|doc| (query, *doc)).collect();
            let encodings = self
                .tokenizer
                .encode_batch(pairs, true)
                .map_err(|e| anyhow::anyhow!(e.to_string()))
                .context("tokenizing rerank batch")?;

            let seq_len = encodings
                .first()
                .context("tokenizer returned an empty batch")?
                .len();
            let batch_size = batch.len();

            let mut input_ids = Vec::with_capacity(batch_size * seq_len);
            let mut attention_mask = Vec::with_capacity(batch_size * seq_len);
            let mut token_type_ids = Vec::with_capacity(batch_size * seq_len);
            for encoding in &encodings {
                input_ids.extend(encoding.get_ids().iter().map(|&id| id as i64));
                attention_mask.extend(encoding.get_attention_mask().iter().map(|&m| m as i64));
                token_type_ids.extend(encoding.get_type_ids().iter().map(|&t| t as i64));
            }

            let shape = [batch_size, seq_len];
            let mut inputs: Vec<(&str, SessionInputValue)> = vec![
                ("input_ids", Tensor::from_array((shape, input_ids))?.into()),
                (
                    "attention_mask",
                    Tensor::from_array((shape, attention_mask))?.into(),
                ),
            ];
            if self.need_token_type_ids {
                inputs.push((
                    "token_type_ids",
                    Tensor::from_array((shape, token_type_ids))?.into(),
                ));
            }

            let outputs = self
                .session
                .run(inputs)
                .context("running onnx rerank session")?;
            let (out_shape, data) = outputs
                .get("logits")
                .context("onnx model output missing 'logits'")?
                .try_extract_tensor::<f32>()
                .context("extracting logits tensor")?;
            // Cross-encoder heads emit one score per row (shape [batch, 1] or
            // [batch, num_labels]); column 0 is the relevance logit in both
            // cases, matching fastembed's own `.slice(s![.., 0])`.
            let num_cols = out_shape.get(1).copied().unwrap_or(1).max(1) as usize;
            for row in 0..batch_size {
                scores.push(data[row * num_cols]);
            }
        }
        Ok(scores)
    }
}

/// Indices of `documents` sorted by length (stable), cut into runs of at most
/// `size`.
fn length_buckets(documents: &[&str], size: usize) -> Vec<Vec<usize>> {
    let mut order: Vec<usize> = (0..documents.len()).collect();
    order.sort_by_key(|&i| documents[i].len());
    order.chunks(size.max(1)).map(<[usize]>::to_vec).collect()
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::model_cache::test_support::CacheDirOverride;

    const JINA_REPO: &str = "models--jinaai--jina-reranker-v1-turbo-en";

    fn jina() -> CachedModel {
        reranker_model(&RerankerModel::JINARerankerV1TurboEn)
    }

    fn seed_files(snap: &Path, files: &[String]) {
        for file in files {
            let path = snap.join(file);
            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
            std::fs::write(path, b"x").unwrap();
        }
    }

    fn seed_snapshot(cache_root: &Path, revision: &str) -> PathBuf {
        let snap = cache_root.join(JINA_REPO).join("snapshots").join(revision);
        std::fs::create_dir_all(&snap).unwrap();
        seed_files(&snap, &jina().files);
        snap
    }

    fn seed_snapshot_without_tokenizer(cache_root: &Path, revision: &str) -> PathBuf {
        let snap = seed_snapshot(cache_root, revision);
        std::fs::remove_file(snap.join("tokenizer.json")).unwrap();
        snap
    }

    #[test]
    fn length_buckets_group_similar_lengths_and_cover_every_document_once() {
        let docs = [
            "a",
            "a much longer document here",
            "bb",
            "ccc",
            "a medium one",
        ];

        let buckets = length_buckets(&docs, 2);

        assert_eq!(buckets, vec![vec![0, 2], vec![3, 4], vec![1]]);
    }

    #[test]
    fn snapshot_dir_finds_an_existing_cached_snapshot() {
        let tmp = tempfile::tempdir().unwrap();
        seed_snapshot(tmp.path(), "abc123");
        let _env = CacheDirOverride::new(tmp.path());

        let found = snapshot_dir(&jina());

        assert_eq!(
            found.unwrap(),
            tmp.path().join(JINA_REPO).join("snapshots/abc123")
        );
    }

    #[test]
    fn snapshot_dir_errors_when_nothing_was_ever_downloaded() {
        let tmp = tempfile::tempdir().unwrap();
        let _env = CacheDirOverride::new(tmp.path());

        let err = snapshot_dir(&jina()).unwrap_err();

        assert!(err.to_string().contains("does not download"), "{err:#}");
    }

    #[test]
    fn snapshot_dir_errors_when_snapshots_dir_exists_but_is_empty() {
        let tmp = tempfile::tempdir().unwrap();
        std::fs::create_dir_all(tmp.path().join(JINA_REPO).join("snapshots")).unwrap();
        let _env = CacheDirOverride::new(tmp.path());

        let err = snapshot_dir(&jina()).unwrap_err();

        assert!(err.to_string().contains("does not download"));
    }

    #[test]
    fn snapshot_dir_errors_when_the_cached_snapshot_lacks_tokenizer_json() {
        let tmp = tempfile::tempdir().unwrap();
        seed_snapshot_without_tokenizer(tmp.path(), "interrupted");
        let _env = CacheDirOverride::new(tmp.path());

        let err = snapshot_dir(&jina()).unwrap_err();

        assert!(err.to_string().contains("does not download"), "{err:#}");
    }

    #[test]
    fn resolve_model_files_with_downloads_once_when_snapshot_missing() {
        let tmp = tempfile::tempdir().unwrap();
        let _env = CacheDirOverride::new(tmp.path());

        let cache_root = tmp.path().to_path_buf();
        let calls = std::cell::Cell::new(0);

        let result = resolve_model_files_with(RerankerModel::JINARerankerV1TurboEn, |_| {
            calls.set(calls.get() + 1);
            seed_snapshot(&cache_root, "downloaded-rev");
            Ok(())
        });

        assert_eq!(calls.get(), 1);
        assert!(result.is_ok());
    }

    #[test]
    fn resolve_model_files_with_skips_download_when_snapshot_already_present() {
        let tmp = tempfile::tempdir().unwrap();
        seed_snapshot(tmp.path(), "already-cached");
        let _env = CacheDirOverride::new(tmp.path());

        let calls = std::cell::Cell::new(0);
        let result = resolve_model_files_with(RerankerModel::JINARerankerV1TurboEn, |_| {
            calls.set(calls.get() + 1);
            Ok(())
        });

        assert_eq!(calls.get(), 0);
        assert!(result.is_ok());
    }

    #[test]
    fn resolve_model_files_with_downloads_again_when_the_cached_snapshot_is_partial() {
        let tmp = tempfile::tempdir().unwrap();
        let snap = seed_snapshot_without_tokenizer(tmp.path(), "rev1");
        let _env = CacheDirOverride::new(tmp.path());

        let calls = std::cell::Cell::new(0);
        let result = resolve_model_files_with(RerankerModel::JINARerankerV1TurboEn, |_| {
            calls.set(calls.get() + 1);
            std::fs::write(snap.join("tokenizer.json"), b"x").unwrap();
            Ok(())
        });

        assert_eq!(calls.get(), 1);
        assert!(result.unwrap().tokenizer_json.is_file());
    }

    #[test]
    fn resolve_model_files_with_propagates_download_error() {
        let tmp = tempfile::tempdir().unwrap();
        let _env = CacheDirOverride::new(tmp.path());

        let result = resolve_model_files_with(RerankerModel::JINARerankerV1TurboEn, |_| {
            anyhow::bail!("network unreachable")
        });

        let err = result.unwrap_err();
        assert!(err.to_string().contains("network unreachable"));
    }
}