candding 0.1.0

Pure candle embeddings for Rust with no ONNX: dense, sparse, multi-vector and rerank models today, image models on the roadmap
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

//! Rejoining WordPiece sub-tokens into whole words, shared by `minicoil` and `bm42`.
//! The `##` marker is hardcoded: every encoder these heads run uses WordPiece.

use std::ops::Range;

const CONTINUATION_PREFIX: &str = "##";

/// A whole word with the span of token positions it came from, for folding per-token data into it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Word {
  pub text: String,
  pub tokens: Range<usize>,
}

/// Rejoins `tokens` into whole words: a `##` token continues the last word, any other starts a new one.
/// A leading `##` token with no word to continue starts its own instead of panicking.
pub fn reconstruct(tokens: &[String]) -> Vec<Word> {
  let mut words: Vec<Word> = Vec::new();
  for (index, token) in tokens.iter().enumerate() {
    match token.strip_prefix(CONTINUATION_PREFIX) {
      Some(continuation) if !words.is_empty() => {
        let word = words.last_mut().expect("just checked words is not empty");
        word.text.push_str(continuation);
        word.tokens.end = index + 1;
      }
      Some(continuation) => words.push(Word {
        text: continuation.to_string(),
        tokens: index..index + 1,
      }),
      None => words.push(Word {
        text: token.clone(),
        tokens: index..index + 1,
      }),
    }
  }
  words
}

#[cfg(test)]
mod tests;