indexlake 0.6.0

An experimental table format with extensible index support.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148

mod definition;
mod manager;

pub use definition::*;
pub use manager::*;
use uuid::Uuid;

use crate::ILResult;
pub use crate::catalog::RowValidity;
use crate::expr::Expr;
use crate::storage::{InputFile, OutputFile};
use arrow::array::{ArrayRef, RecordBatch};
use arrow::datatypes::FieldRef;
use std::any::Any;
use std::fmt::Debug;
use std::sync::Arc;

pub trait IndexKind: Debug + Send + Sync {
    // The kind of the index.
    fn kind(&self) -> &str;

    fn decode_params(&self, value: &str) -> ILResult<Arc<dyn IndexParams>>;

    fn supports(&self, index_def: &IndexDefinition) -> ILResult<()>;

    fn builder(&self, index_def: &IndexDefinitionRef) -> ILResult<Box<dyn IndexBuilder>>;

    fn supports_search(
        &self,
        index_def: &IndexDefinition,
        query: &dyn SearchQuery,
    ) -> ILResult<bool>;

    fn search_query_codec(&self) -> Option<Arc<dyn SearchQueryCodec>>;

    fn dynamic_fields(&self) -> Vec<FieldRef>;

    fn supports_filter(
        &self,
        index_def: &IndexDefinition,
        filter: &Expr,
    ) -> ILResult<FilterSupport>;

    /// Number of rows per inline index segment for this index kind.
    /// Implementations should inspect [`IndexDefinition`] (e.g. key column type,
    /// vector dimension) to produce a reasonable value.
    fn inline_index_segment_row_count(&self, index_def: &IndexDefinition) -> usize;
}

#[async_trait::async_trait]
pub trait IndexBuilder: Debug + Send + Sync {
    fn index_def(&self) -> &IndexDefinitionRef;

    fn is_empty(&self) -> bool;

    fn append(&mut self, batch: &RecordBatch) -> ILResult<()>;

    async fn read_file(&mut self, input_file: Box<dyn InputFile>) -> ILResult<()>;

    async fn write_file(&mut self, output_file: Box<dyn OutputFile>) -> ILResult<()>;

    fn read_bytes(&mut self, buf: &[u8]) -> ILResult<()>;

    fn write_bytes(&mut self, buf: &mut Vec<u8>) -> ILResult<()>;

    fn build(&mut self) -> ILResult<Box<dyn Index>>;
}

#[async_trait::async_trait]
pub trait Index: Debug + Send + Sync {
    /// Search the index.
    /// `validity` indicates which rows in the index are valid.
    /// Index implementations must filter out invalid rows internally
    /// so that they do not occupy result slots.
    async fn search(
        &self,
        query: &dyn SearchQuery,
        dynamic_fields: &[String],
        validity: &RowValidity,
        limit: Option<usize>,
    ) -> ILResult<SearchIndexEntries>;

    async fn filter(
        &self,
        filters: &[Expr],
        validity: &RowValidity,
    ) -> ILResult<FilterIndexEntries>;
}

#[derive(Debug, Clone)]
pub struct DynamicColumn {
    pub field: FieldRef,
    pub values: ArrayRef,
}

#[derive(Debug, Clone)]
pub struct SearchIndexEntries {
    pub row_ids: Vec<Uuid>,
    pub scores: Vec<f64>,
    pub score_higher_is_better: bool,
    pub dynamic_columns: Vec<DynamicColumn>,
}

#[derive(Debug, Clone)]
pub struct FilterIndexEntries {
    pub row_ids: Vec<Uuid>,
}

#[derive(Debug, Clone, Copy)]
pub enum FilterSupport {
    Unsupported,
    Exact,
    Inexact,
}

pub trait SearchQuery: Debug + Send + Sync + Any {
    fn index_kind(&self) -> &str;
}

/// Codec for serializing/deserializing SearchQuery to/from bytes.
pub trait SearchQueryCodec: Debug + Send + Sync {
    fn encode(&self, query: &dyn SearchQuery) -> ILResult<Vec<u8>>;
    fn decode(&self, data: &[u8]) -> ILResult<Arc<dyn SearchQuery>>;
}

impl dyn SearchQuery {
    pub fn is<T: SearchQuery>(&self) -> bool {
        (self as &dyn Any).is::<T>()
    }

    pub fn downcast_ref<T: SearchQuery>(&self) -> Option<&T> {
        (self as &dyn Any).downcast_ref::<T>()
    }
}

pub trait IndexParams: Debug + Send + Sync + Any {
    fn encode(&self) -> ILResult<String>;
}

impl dyn IndexParams {
    pub fn is<T: IndexParams>(&self) -> bool {
        (self as &dyn Any).is::<T>()
    }

    pub fn downcast_ref<T: IndexParams>(&self) -> Option<&T> {
        (self as &dyn Any).downcast_ref::<T>()
    }
}