Struct InvertedIndex 

pub struct InvertedIndex { /* private fields */ }

Implementations

impl InvertedIndex

pub fn tokenizer(&self) -> Box<dyn LanceTokenizer>

pub fn params(&self) -> &InvertedIndexParams

pub fn partition_count(&self) -> usize

Returns the number of partitions in this inverted index.

pub fn deleted_fragments(&self) -> &RoaringBitmap

Returns the set of fragments which are contained in the index, but no longer in the dataset.

Most other indices remove data from deleted fragments when the index updates (copy-on-write). However, this would require an expensive copy of the FTS index. Instead, we track the deleted fragments and prune them at search time (merge-on-read).

pub async fn merge_segments( segments: &[Arc<Self>], new_data: SendableRecordBatchStream, dest_store: &dyn IndexStore, old_data_filter: Option<OldIndexDataFilter>, progress: Arc<dyn IndexBuildProgress>, ) -> Result<CreatedIndex>

pub fn bm25_base_scorer(&self, query_tokens: &Tokens) -> MemBM25Scorer

pub fn bm25_stats_for_terms(&self, terms: &[String]) -> (u64, usize, Vec<usize>)

pub fn expand_fuzzy_tokens( &self, tokens: &Tokens, params: &FtsSearchParams, ) -> Result<Tokens>

Expand fuzzy query tokens against all partitions in this segment.

pub async fn bm25_search( &self, tokens: Arc<Tokens>, params: Arc<FtsSearchParams>, operator: Operator, prefilter: Arc<dyn PreFilter>, metrics: Arc<dyn MetricsCollector>, base_scorer: Option<&MemBM25Scorer>, ) -> Result<(Vec<u64>, Vec<f32>)>

Search documents that match the query and return row ids sorted by BM25 score.

When base_scorer is provided, search uses those corpus-level BM25 statistics instead of deriving them from this segment alone.

pub fn is_legacy(&self) -> bool

pub async fn load( store: Arc<dyn IndexStore>, frag_reuse_index: Option<Arc<FragReuseIndex>>, index_cache: &LanceCache, ) -> Result<Arc<Self>>

where Self: Sized,

impl InvertedIndex

pub async fn prewarm_with_options( &self, options: &FtsPrewarmOptions, ) -> Result<()>

Trait Implementations

impl Clone for InvertedIndex

fn clone(&self) -> InvertedIndex

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for InvertedIndex

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl DeepSizeOf for InvertedIndex

fn deep_size_of_children(&self, context: &mut Context) -> usize

Returns an estimation of the heap-managed storage of this object. This does not include the size of the object itself.

fn deep_size_of(&self) -> usize

Returns an estimation of a total size of memory owned by the object, including heap-managed storage.

impl Index for InvertedIndex

fn as_any(&self) -> &dyn Any

Cast to Any.

fn as_index(self: Arc<Self>) -> Arc<dyn Index>

Cast to Index

fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>>

Cast to vector::VectorIndex

fn statistics(&self) -> Result<Value>

Retrieve index statistics as a JSON Value

fn prewarm<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Prewarm the index.

fn index_type(&self) -> IndexType

Get the type of the index

fn calculate_included_frags<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<RoaringBitmap>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Read through the index and determine which fragment ids are covered by the index

impl ScalarIndex for InvertedIndex

fn search<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, query: &'life1 dyn AnyQuery, _metrics: &'life2 dyn MetricsCollector, ) -> Pin<Box<dyn Future<Output = Result<SearchResult>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Search the scalar index

fn can_remap(&self) -> bool

Returns true if the remap operation is supported

fn remap<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, mapping: &'life1 HashMap<u64, Option<u64>>, dest_store: &'life2 dyn IndexStore, ) -> Pin<Box<dyn Future<Output = Result<CreatedIndex>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Remap the row ids, creating a new remapped version of this index in dest_store

fn update<'life0, 'life1, 'async_trait>( &'life0 self, new_data: SendableRecordBatchStream, dest_store: &'life1 dyn IndexStore, old_data_filter: Option<OldIndexDataFilter>, ) -> Pin<Box<dyn Future<Output = Result<CreatedIndex>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Add the new data into the index, creating an updated version of the index in dest_store

fn update_criteria(&self) -> UpdateCriteria

Returns the criteria that will be used to update the index

fn derive_index_params(&self) -> Result<ScalarIndexParams>

Derive the index parameters from the current index