mnestic 0.8.2

A transactional relational-graph-vector database using Datalog — a maintained fork of CozoDB, tuned as a substrate for agentic memory
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

use crate::fts::tokenizer::Token;

#[derive(Debug)]
pub(crate) struct CangjieTokenStream<'a> {
    result: Vec<&'a str>,
    // Begin with 1
    index: usize,
    offset_from: usize,
    token: Token,
}

impl<'a> CangjieTokenStream<'a> {
    pub(crate) fn new(result: Vec<&'a str>) -> Self {
        CangjieTokenStream {
            result,
            index: 0,
            offset_from: 0,
            token: Token::default(),
        }
    }
}

impl<'a> crate::fts::tokenizer::TokenStream for CangjieTokenStream<'a> {
    fn advance(&mut self) -> bool {
        if self.index < self.result.len() {
            let current_word = self.result[self.index];
            let offset_to = self.offset_from + current_word.len();

            self.token = Token {
                offset_from: self.offset_from,
                offset_to,
                position: self.index,
                text: current_word.to_string(),
                position_length: self.result.len(),
            };

            self.index += 1;
            self.offset_from = offset_to;
            true
        } else {
            false
        }
    }

    fn token(&self) -> &crate::fts::tokenizer::Token {
        &self.token
    }

    fn token_mut(&mut self) -> &mut crate::fts::tokenizer::Token {
        &mut self.token
    }
}