luckyshot 0.5.1

A CLI tool for one-shot code generations using RAG and file watching
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

use crate::tokenizer::get_tokenizer;
use bm25::{DefaultTokenizer, EmbedderBuilder};

#[derive(Debug)]
pub struct Bm25Vector {
    pub indices: Vec<u32>,
    pub values: Vec<f32>,
}

pub fn create_bm25_vector(text: &str, avgdl: f32) -> Bm25Vector {
    let mut embedder_builder = EmbedderBuilder::<u32, DefaultTokenizer>::with_avgdl(avgdl);
    embedder_builder = embedder_builder.tokenizer(get_tokenizer());
    let embedder = embedder_builder.build();

    let embedding = embedder.embed(text);
    Bm25Vector {
        indices: embedding.indices().cloned().collect(),
        values: embedding.values().cloned().collect(),
    }
}