indexlake 0.4.0

An experimental table format with extensible index support.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

mod definition;
mod manager;

pub use definition::*;
pub use manager::*;
use uuid::Uuid;

use crate::ILResult;
use crate::expr::Expr;
use crate::storage::{InputFile, OutputFile};
use arrow::array::RecordBatch;
use std::any::Any;
use std::fmt::Debug;
use std::sync::Arc;

pub trait IndexKind: Debug + Send + Sync {
    // The kind of the index.
    fn kind(&self) -> &str;

    fn decode_params(&self, value: &str) -> ILResult<Arc<dyn IndexParams>>;

    fn supports(&self, index_def: &IndexDefinition) -> ILResult<()>;

    fn builder(&self, index_def: &IndexDefinitionRef) -> ILResult<Box<dyn IndexBuilder>>;

    fn supports_search(
        &self,
        index_def: &IndexDefinition,
        query: &dyn SearchQuery,
    ) -> ILResult<bool>;

    fn supports_filter(
        &self,
        index_def: &IndexDefinition,
        filter: &Expr,
    ) -> ILResult<FilterSupport>;
}

#[async_trait::async_trait]
pub trait IndexBuilder: Debug + Send + Sync {
    fn index_def(&self) -> &IndexDefinitionRef;

    fn append(&mut self, batch: &RecordBatch) -> ILResult<()>;

    async fn read_file(&mut self, input_file: Box<dyn InputFile>) -> ILResult<()>;

    async fn write_file(&mut self, output_file: Box<dyn OutputFile>) -> ILResult<()>;

    fn read_bytes(&mut self, buf: &[u8]) -> ILResult<()>;

    fn write_bytes(&mut self, buf: &mut Vec<u8>) -> ILResult<()>;

    fn build(&mut self) -> ILResult<Box<dyn Index>>;
}

#[async_trait::async_trait]
pub trait Index: Debug + Send + Sync {
    async fn search(&self, query: &dyn SearchQuery) -> ILResult<SearchIndexEntries>;

    async fn filter(&self, filters: &[Expr]) -> ILResult<FilterIndexEntries>;
}

#[derive(Debug, Clone)]
pub struct SearchIndexEntries {
    pub row_id_scores: Vec<RowIdScore>,
    pub score_higher_is_better: bool,
}

#[derive(Debug, Clone)]
pub struct RowIdScore {
    pub row_id: Uuid,
    pub score: f64,
}

#[derive(Debug, Clone)]
pub struct FilterIndexEntries {
    pub row_ids: Vec<Uuid>,
}

#[derive(Debug, Clone, Copy)]
pub enum FilterSupport {
    Unsupported,
    Exact,
    Inexact,
}

pub trait SearchQuery: Debug + Send + Sync {
    fn as_any(&self) -> &dyn Any;

    fn index_kind(&self) -> &str;

    fn limit(&self) -> Option<usize>;
}

pub trait IndexParams: Debug + Send + Sync {
    fn as_any(&self) -> &dyn Any;

    fn encode(&self) -> ILResult<String>;
}