luckyshot 0.5.1

A CLI tool for one-shot code generations using RAG and file watching
1
2
3
4
5
6
7
8
9
10

use bm25::{DefaultTokenizer, Language};

pub fn get_tokenizer() -> DefaultTokenizer {
    DefaultTokenizer::builder()
        .language_mode(Language::English)
        .normalization(true) // Normalize unicode (e.g., 'é' -> 'e', '🍕' -> 'pizza', etc.)
        .stopwords(true) // Remove common words with little meaning (e.g., 'the', 'and', 'of', etc.)
        .stemming(true) // Reduce words to their root form (e.g., 'running' -> 'run')
        .build()
}