memd 1.9.0

Local memory CLI for AI agents with hybrid retrieval
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
//! Code-aware tokenization for hybrid search.
//!
//! This module provides tokenization that handles both code identifiers
//! (camelCase, snake_case splitting) and prose (stemming/normalization).

use rust_stemmers::{Algorithm, Stemmer};
use std::sync::Arc;
use tantivy::tokenizer::{Token, TokenStream, Tokenizer};

/// Type of content being tokenized.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenType {
    /// Code identifiers and keywords
    Code,
    /// Natural language prose
    Prose,
    /// Mixed code and prose
    Mixed,
}

/// A token with type information.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TypedToken {
    pub text: String,
    pub token_type: TokenType,
    pub offset_from: usize,
    pub offset_to: usize,
}

/// Code-aware tokenizer that splits identifiers and normalizes prose.
///
/// Features:
/// - Splits camelCase: "parseUserInput" -> ["parse", "User", "Input"]
/// - Splits snake_case: "parse_user_input" -> ["parse", "user", "input"]
/// - Preserves acronyms: "HTTPResponse" -> ["HTTP", "Response"]
/// - Stems prose tokens using Porter algorithm
#[derive(Clone)]
pub struct CodeTokenizer {
    stemmer: Arc<Stemmer>,
}

impl Default for CodeTokenizer {
    fn default() -> Self {
        Self::new()
    }
}

impl CodeTokenizer {
    /// Create a new code tokenizer with English stemmer.
    pub fn new() -> Self {
        Self {
            stemmer: Arc::new(Stemmer::create(Algorithm::English)),
        }
    }

    /// Tokenize text into typed tokens.
    pub fn tokenize_typed(&self, text: &str) -> Vec<TypedToken> {
        let mut tokens = Vec::new();
        let mut word_start: Option<usize> = None;

        for (idx, ch) in text.char_indices() {
            let is_delimiter = ch.is_whitespace() || ch == '_' || is_separator(ch);
            if is_delimiter {
                if let Some(start) = word_start.take() {
                    self.push_word_tokens(&mut tokens, &text[start..idx], start);
                }
            } else if word_start.is_none() {
                word_start = Some(idx);
            }
        }

        if let Some(start) = word_start {
            self.push_word_tokens(&mut tokens, &text[start..], start);
        }

        tokens
    }

    /// Tokenize text into simple string tokens (lowercased, stemmed).
    pub fn tokenize(&self, text: &str) -> Vec<String> {
        self.tokenize_typed(text)
            .into_iter()
            .map(|t| t.text)
            .collect()
    }

    fn push_word_tokens(&self, tokens: &mut Vec<TypedToken>, word: &str, word_start: usize) {
        let subtokens = split_camel_case(word);
        let mut subtoken_pos = word_start;

        for subtoken in subtokens {
            if subtoken.is_empty() {
                continue;
            }

            let subtoken_end = subtoken_pos + subtoken.len();
            let token_type = detect_token_type(&subtoken);
            let normalized = match token_type {
                TokenType::Code => {
                    if is_acronym(&subtoken) {
                        subtoken.clone()
                    } else {
                        subtoken.to_lowercase()
                    }
                }
                TokenType::Prose => {
                    let lower = subtoken.to_lowercase();
                    self.stemmer.stem(&lower).to_string()
                }
                TokenType::Mixed => subtoken.to_lowercase(),
            };

            tokens.push(TypedToken {
                text: normalized,
                token_type,
                offset_from: subtoken_pos,
                offset_to: subtoken_end,
            });
            subtoken_pos = subtoken_end;
        }
    }
}

/// Split a word on camelCase boundaries.
///
/// Examples:
/// - "getUserById" -> ["get", "User", "By", "Id"]
/// - "HTTPResponse" -> ["HTTP", "Response"]
/// - "parseJSONData" -> ["parse", "JSON", "Data"]
fn split_camel_case(word: &str) -> Vec<String> {
    let mut result = Vec::new();
    let mut current = String::new();
    let chars: Vec<char> = word.chars().collect();

    for i in 0..chars.len() {
        let c = chars[i];

        if current.is_empty() {
            current.push(c);
            continue;
        }

        let prev_upper = chars
            .get(i.saturating_sub(1))
            .is_some_and(|c| c.is_uppercase());
        let curr_upper = c.is_uppercase();
        let next_lower = chars.get(i + 1).is_some_and(|c| c.is_lowercase());

        // Split cases:
        // 1. lowercase -> uppercase: "getUser" at 'U'
        // 2. uppercase -> uppercase -> lowercase: "HTTPResponse" at 'R' (keep HTTP together)
        let should_split = if !prev_upper && curr_upper {
            // Case 1: lowercase to uppercase
            true
        } else if prev_upper && curr_upper && next_lower {
            // Case 2: end of acronym (e.g., "HTTPResponse")
            true
        } else {
            false
        };

        if should_split {
            if !current.is_empty() {
                result.push(current);
            }
            current = String::new();
        }

        current.push(c);
    }

    if !current.is_empty() {
        result.push(current);
    }

    result
}

/// Check if a string is an acronym (2+ consecutive uppercase letters).
fn is_acronym(s: &str) -> bool {
    let chars: Vec<char> = s.chars().collect();
    chars.len() >= 2 && chars.iter().all(|c| c.is_uppercase())
}

/// Check if a character is a separator (not whitespace or underscore).
fn is_separator(c: char) -> bool {
    matches!(
        c,
        '.' | ','
            | ';'
            | ':'
            | '('
            | ')'
            | '['
            | ']'
            | '{'
            | '}'
            | '<'
            | '>'
            | '"'
            | '\''
            | '&'
            | '|'
            | '+'
            | '-'
            | '*'
            | '/'
            | '%'
            | '='
            | '!'
            | '?'
    )
}

/// Detect the token type based on content.
fn detect_token_type(token: &str) -> TokenType {
    let has_upper = token.chars().any(|c| c.is_uppercase());
    let has_lower = token.chars().any(|c| c.is_lowercase());
    let all_alpha = token.chars().all(|c| c.is_alphabetic());

    if !all_alpha {
        return TokenType::Code;
    }

    if is_acronym(token) {
        return TokenType::Code;
    }

    if has_upper && has_lower {
        // Mixed case like "User" or "Input"
        TokenType::Code
    } else if has_lower && !has_upper {
        // All lowercase - likely prose
        TokenType::Prose
    } else {
        TokenType::Mixed
    }
}

// Tantivy Tokenizer trait implementation

impl Tokenizer for CodeTokenizer {
    type TokenStream<'a> = CodeTokenStream;

    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
        let tokens = self.tokenize_typed(text);
        CodeTokenStream::new(tokens)
    }
}

/// Token stream for tantivy integration.
pub struct CodeTokenStream {
    tokens: Vec<TypedToken>,
    index: usize,
    token: Token,
}

impl CodeTokenStream {
    fn new(tokens: Vec<TypedToken>) -> Self {
        Self {
            tokens,
            index: 0,
            token: Token::default(),
        }
    }
}

impl TokenStream for CodeTokenStream {
    fn advance(&mut self) -> bool {
        if self.index >= self.tokens.len() {
            return false;
        }

        let typed = &self.tokens[self.index];
        self.token.text.clear();
        self.token.text.push_str(&typed.text);
        self.token.offset_from = typed.offset_from;
        self.token.offset_to = typed.offset_to;
        self.token.position = self.index;
        self.token.position_length = 1;

        self.index += 1;
        true
    }

    fn token(&self) -> &Token {
        &self.token
    }

    fn token_mut(&mut self) -> &mut Token {
        &mut self.token
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_camel_case_splitting() {
        let tokenizer = CodeTokenizer::new();

        // Basic camelCase
        let tokens = tokenizer.tokenize("getUserById");
        assert_eq!(tokens, vec!["get", "user", "by", "id"]);

        // Multiple words
        let tokens = tokenizer.tokenize("parseUserInput");
        assert_eq!(tokens, vec!["pars", "user", "input"]); // "parse" stems to "pars"
    }

    #[test]
    fn test_snake_case_splitting() {
        let tokenizer = CodeTokenizer::new();

        let tokens = tokenizer.tokenize("get_user_by_id");
        assert_eq!(tokens, vec!["get", "user", "by", "id"]);

        let tokens = tokenizer.tokenize("parse_user_input");
        assert_eq!(tokens, vec!["pars", "user", "input"]);
    }

    #[test]
    fn test_acronym_preservation() {
        let tokenizer = CodeTokenizer::new();

        // Acronyms should be preserved uppercase
        let tokens = tokenizer.tokenize("HTTPResponse");
        assert_eq!(tokens, vec!["HTTP", "response"]);

        let tokens = tokenizer.tokenize("parseJSONData");
        assert_eq!(tokens, vec!["pars", "JSON", "data"]);

        // API, SQL should stay uppercase
        let tokens = tokenizer.tokenize("getAPIKey");
        assert_eq!(tokens, vec!["get", "API", "key"]);
    }

    #[test]
    fn test_prose_normalization() {
        let tokenizer = CodeTokenizer::new();

        // Stemming should apply
        let tokens = tokenizer.tokenize("running quickly");
        assert_eq!(tokens, vec!["run", "quick"]);

        let tokens = tokenizer.tokenize("the users are connecting");
        assert_eq!(tokens, vec!["the", "user", "are", "connect"]);
    }

    #[test]
    fn test_operator_preservation() {
        let tokenizer = CodeTokenizer::new();

        // Operators should be split out (as separators)
        let tokens = tokenizer.tokenize("x && y");
        assert_eq!(tokens, vec!["x", "y"]); // && is filtered as separator
    }

    #[test]
    fn test_mixed_identifiers() {
        let tokenizer = CodeTokenizer::new();

        let tokens = tokenizer.tokenize("parseJSONData");
        assert_eq!(tokens, vec!["pars", "JSON", "data"]);

        let tokens = tokenizer.tokenize("XMLHttpRequest");
        assert_eq!(tokens, vec!["XML", "http", "request"]);
    }

    #[test]
    fn test_typed_tokens() {
        let tokenizer = CodeTokenizer::new();

        let typed = tokenizer.tokenize_typed("HTTPResponse");
        assert_eq!(typed.len(), 2);
        assert_eq!(typed[0].text, "HTTP");
        assert_eq!(typed[0].token_type, TokenType::Code);
        assert_eq!(typed[1].text, "response");
        assert_eq!(typed[1].token_type, TokenType::Code);
    }

    #[test]
    fn test_tantivy_tokenizer_trait() {
        use tantivy::tokenizer::Tokenizer;

        let mut tokenizer = CodeTokenizer::new();
        let mut stream = tokenizer.token_stream("getUserById");

        let mut tokens = Vec::new();
        while stream.advance() {
            tokens.push(stream.token().text.clone());
        }

        assert_eq!(tokens, vec!["get", "user", "by", "id"]);
    }

    #[test]
    fn test_unicode_whitespace_does_not_panic() {
        let tokenizer = CodeTokenizer::new();
        let text = "result\u{200a}value with\u{2003}unicode separators";
        let tokens = tokenizer.tokenize(text);
        assert!(!tokens.is_empty());
        assert!(tokens.contains(&"result".to_string()));
        assert!(tokens.contains(&"valu".to_string()));
    }
}