memd 1.9.0

Local memory CLI for AI agents with hybrid retrieval
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
//! WAL reader for crash recovery
//!
//! Reads WAL records and supports replaying for recovery.
//! Handles partial/corrupt records gracefully (stops at first error).

use std::fs::File;
use std::io::{self, Read};
use std::path::Path;

use super::format::{WalRecord, WalRecordType, WAL_HEADER_SIZE};

/// WAL reader
pub struct WalReader {
    data: Vec<u8>,
}

impl WalReader {
    /// Open and read entire WAL file
    ///
    /// Returns empty reader if file doesn't exist (no records to recover).
    pub fn open(path: &Path) -> io::Result<Self> {
        if !path.exists() {
            return Ok(Self { data: Vec::new() });
        }

        let mut file = File::open(path)?;
        let mut data = Vec::new();
        file.read_to_end(&mut data)?;

        Ok(Self { data })
    }

    /// Check if WAL is empty (no records)
    pub fn is_empty(&self) -> bool {
        self.data.is_empty()
    }

    /// Get raw data length
    pub fn len(&self) -> usize {
        self.data.len()
    }

    /// Read all valid records from WAL
    ///
    /// Stops at first corrupted record (partial write on crash).
    /// Returns records and count of bytes successfully read.
    pub fn read_all_records(&self) -> io::Result<(Vec<WalRecord>, usize)> {
        let mut records = Vec::new();
        let mut offset = 0;

        while offset + WAL_HEADER_SIZE <= self.data.len() {
            match WalRecord::decode_from_bytes(&self.data[offset..]) {
                Ok((record, consumed)) => {
                    records.push(record);
                    offset += consumed;
                }
                Err(_) => {
                    // Corrupted record - stop here (partial write on crash)
                    break;
                }
            }
        }

        Ok((records, offset))
    }

    /// Find the index of the last checkpoint record
    pub fn find_last_checkpoint(&self) -> io::Result<Option<usize>> {
        let (records, _) = self.read_all_records()?;

        Ok(records
            .iter()
            .rposition(|r| matches!(r.record_type, WalRecordType::Checkpoint)))
    }

    /// Get records for recovery (after last checkpoint)
    ///
    /// Returns records that need to be replayed.
    pub fn records_for_recovery(&self) -> io::Result<Vec<WalRecord>> {
        let (records, _) = self.read_all_records()?;

        let checkpoint_idx = records
            .iter()
            .rposition(|r| matches!(r.record_type, WalRecordType::Checkpoint))
            .map(|i| i + 1) // Start after checkpoint
            .unwrap_or(0); // No checkpoint = replay all

        Ok(records[checkpoint_idx..].to_vec())
    }
}

/// Recovery helper functions
pub mod recovery {
    use super::*;

    /// Callback trait for applying recovered records
    pub trait RecoveryHandler {
        /// Called for each Add record
        fn on_add(&mut self, record: &WalRecord) -> io::Result<()>;

        /// Called for each Delete record
        fn on_delete(&mut self, record: &WalRecord) -> io::Result<()>;

        /// Called for each TaskArtifact record.
        fn on_task_artifact(&mut self, record: &WalRecord) -> io::Result<()>;

        /// Check if a chunk already exists (for idempotency)
        fn chunk_exists(&self, chunk_id: &str) -> io::Result<bool>;
    }

    /// Recovery statistics
    #[derive(Debug, Default, Clone, PartialEq, Eq)]
    pub struct RecoveryStats {
        /// Number of Add records applied
        pub adds: usize,
        /// Number of Delete records applied
        pub deletes: usize,
        /// Number of task artifact records applied
        pub task_artifacts: usize,
        /// Number of records skipped (already exist)
        pub skipped: usize,
        /// Number of checkpoint records encountered (should be 0 after last checkpoint)
        pub checkpoints: usize,
    }

    /// Replay WAL records through handler
    ///
    /// Skips Add records for chunks that already exist (idempotent).
    pub fn replay<H: RecoveryHandler>(
        reader: &WalReader,
        handler: &mut H,
    ) -> io::Result<RecoveryStats> {
        let records = reader.records_for_recovery()?;

        let mut stats = RecoveryStats::default();

        for record in &records {
            match record.record_type {
                WalRecordType::Add => {
                    // Idempotent: skip if chunk already exists
                    if handler.chunk_exists(&record.chunk_id)? {
                        stats.skipped += 1;
                        continue;
                    }
                    handler.on_add(record)?;
                    stats.adds += 1;
                }
                WalRecordType::Delete => {
                    handler.on_delete(record)?;
                    stats.deletes += 1;
                }
                WalRecordType::TaskArtifact => {
                    handler.on_task_artifact(record)?;
                    stats.task_artifacts += 1;
                }
                WalRecordType::Checkpoint => {
                    // Should not appear after last checkpoint
                    stats.checkpoints += 1;
                }
            }
        }

        Ok(stats)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::store::wal::WalWriter;
    use std::collections::HashSet;
    use tempfile::tempdir;

    #[test]
    fn read_empty_wal() {
        let temp_dir = tempdir().unwrap();
        let wal_path = temp_dir.path().join("nonexistent.wal");

        let reader = WalReader::open(&wal_path).unwrap();
        assert!(reader.is_empty());
        assert_eq!(reader.len(), 0);

        let (records, bytes_read) = reader.read_all_records().unwrap();
        assert!(records.is_empty());
        assert_eq!(bytes_read, 0);
    }

    #[test]
    fn read_written_records() {
        let temp_dir = tempdir().unwrap();
        let wal_path = temp_dir.path().join("test.wal");

        // Write records
        {
            let mut writer = WalWriter::create(&wal_path).unwrap();
            writer
                .append_add("tenant_1", "chunk_1", 1000, b"data1".to_vec())
                .unwrap();
            writer
                .append_add("tenant_1", "chunk_2", 2000, b"data2".to_vec())
                .unwrap();
            writer.append_delete("tenant_1", "chunk_1", 3000).unwrap();
        }

        // Read them back
        let reader = WalReader::open(&wal_path).unwrap();
        assert!(!reader.is_empty());

        let (records, _) = reader.read_all_records().unwrap();
        assert_eq!(records.len(), 3);

        assert_eq!(records[0].record_type, WalRecordType::Add);
        assert_eq!(records[0].chunk_id, "chunk_1");
        assert_eq!(records[0].payload, b"data1");

        assert_eq!(records[1].record_type, WalRecordType::Add);
        assert_eq!(records[1].chunk_id, "chunk_2");

        assert_eq!(records[2].record_type, WalRecordType::Delete);
        assert_eq!(records[2].chunk_id, "chunk_1");
    }

    #[test]
    fn find_checkpoint() {
        let temp_dir = tempdir().unwrap();
        let wal_path = temp_dir.path().join("checkpoint.wal");

        // Write records with checkpoint
        {
            let mut writer = WalWriter::create(&wal_path).unwrap();
            writer.append_add("t1", "c1", 1000, b"d1".to_vec()).unwrap();
            writer.append_checkpoint("t1", 2000).unwrap();
            writer.append_add("t1", "c2", 3000, b"d2".to_vec()).unwrap();
        }

        let reader = WalReader::open(&wal_path).unwrap();
        let checkpoint_idx = reader.find_last_checkpoint().unwrap();
        assert_eq!(checkpoint_idx, Some(1)); // Index of checkpoint record
    }

    #[test]
    fn records_for_recovery() {
        let temp_dir = tempdir().unwrap();
        let wal_path = temp_dir.path().join("recovery.wal");

        // Write records with checkpoint in middle
        {
            let mut writer = WalWriter::create(&wal_path).unwrap();
            writer.append_add("t1", "c1", 1000, b"d1".to_vec()).unwrap();
            writer.append_add("t1", "c2", 2000, b"d2".to_vec()).unwrap();
            writer.append_checkpoint("t1", 3000).unwrap();
            writer.append_add("t1", "c3", 4000, b"d3".to_vec()).unwrap();
            writer.append_add("t1", "c4", 5000, b"d4".to_vec()).unwrap();
        }

        let reader = WalReader::open(&wal_path).unwrap();
        let recovery_records = reader.records_for_recovery().unwrap();

        // Should only have records after checkpoint
        assert_eq!(recovery_records.len(), 2);
        assert_eq!(recovery_records[0].chunk_id, "c3");
        assert_eq!(recovery_records[1].chunk_id, "c4");
    }

    #[test]
    fn records_for_recovery_no_checkpoint() {
        let temp_dir = tempdir().unwrap();
        let wal_path = temp_dir.path().join("no_checkpoint.wal");

        // Write records without checkpoint
        {
            let mut writer = WalWriter::create(&wal_path).unwrap();
            writer.append_add("t1", "c1", 1000, b"d1".to_vec()).unwrap();
            writer.append_add("t1", "c2", 2000, b"d2".to_vec()).unwrap();
        }

        let reader = WalReader::open(&wal_path).unwrap();
        let recovery_records = reader.records_for_recovery().unwrap();

        // All records should be returned
        assert_eq!(recovery_records.len(), 2);
    }

    #[test]
    fn partial_record_tolerance() {
        let temp_dir = tempdir().unwrap();
        let wal_path = temp_dir.path().join("partial.wal");

        // Write records
        {
            let mut writer = WalWriter::create(&wal_path).unwrap();
            writer
                .append_add("t1", "c1", 1000, b"good1".to_vec())
                .unwrap();
            writer
                .append_add("t1", "c2", 2000, b"good2".to_vec())
                .unwrap();
        }

        // Simulate partial write by appending garbage
        {
            use std::io::Write;
            let mut file = std::fs::OpenOptions::new()
                .append(true)
                .open(&wal_path)
                .unwrap();
            // Write partial header (not a complete record)
            file.write_all(b"MWAL").unwrap(); // Magic only, incomplete header
            file.write_all(&[1, 0, 0]).unwrap(); // Partial length
        }

        // Reader should return only the valid records
        let reader = WalReader::open(&wal_path).unwrap();
        let (records, bytes_read) = reader.read_all_records().unwrap();

        assert_eq!(records.len(), 2);
        assert_eq!(records[0].chunk_id, "c1");
        assert_eq!(records[1].chunk_id, "c2");
        // bytes_read should be less than total file size
        assert!(bytes_read < reader.len());
    }

    #[test]
    fn recovery_replay_idempotent() {
        use recovery::{replay, RecoveryHandler, RecoveryStats};

        // Mock handler that tracks calls
        struct MockHandler {
            existing: HashSet<String>,
            added: Vec<String>,
            deleted: Vec<String>,
        }

        impl RecoveryHandler for MockHandler {
            fn on_add(&mut self, record: &WalRecord) -> io::Result<()> {
                self.added.push(record.chunk_id.clone());
                self.existing.insert(record.chunk_id.clone());
                Ok(())
            }

            fn on_delete(&mut self, record: &WalRecord) -> io::Result<()> {
                self.deleted.push(record.chunk_id.clone());
                Ok(())
            }

            fn on_task_artifact(&mut self, _record: &WalRecord) -> io::Result<()> {
                Ok(())
            }

            fn chunk_exists(&self, chunk_id: &str) -> io::Result<bool> {
                Ok(self.existing.contains(chunk_id))
            }
        }

        let temp_dir = tempdir().unwrap();
        let wal_path = temp_dir.path().join("replay.wal");

        // Write WAL
        {
            let mut writer = WalWriter::create(&wal_path).unwrap();
            writer
                .append_add("t1", "new_chunk", 1000, b"data".to_vec())
                .unwrap();
            writer
                .append_add("t1", "existing_chunk", 2000, b"data".to_vec())
                .unwrap();
            writer.append_delete("t1", "to_delete", 3000).unwrap();
        }

        let reader = WalReader::open(&wal_path).unwrap();

        // First replay - existing_chunk already exists
        let mut handler = MockHandler {
            existing: HashSet::from(["existing_chunk".to_string()]),
            added: Vec::new(),
            deleted: Vec::new(),
        };

        let stats = replay(&reader, &mut handler).unwrap();

        assert_eq!(
            stats,
            RecoveryStats {
                adds: 1,    // Only new_chunk added
                skipped: 1, // existing_chunk skipped
                deletes: 1, // to_delete processed
                task_artifacts: 0,
                checkpoints: 0,
            }
        );

        assert_eq!(handler.added, vec!["new_chunk"]);
        assert_eq!(handler.deleted, vec!["to_delete"]);
    }

    #[test]
    fn recovery_after_checkpoint() {
        use recovery::{replay, RecoveryHandler};

        struct CountingHandler {
            add_count: usize,
        }

        impl RecoveryHandler for CountingHandler {
            fn on_add(&mut self, _record: &WalRecord) -> io::Result<()> {
                self.add_count += 1;
                Ok(())
            }

            fn on_delete(&mut self, _record: &WalRecord) -> io::Result<()> {
                Ok(())
            }

            fn on_task_artifact(&mut self, _record: &WalRecord) -> io::Result<()> {
                Ok(())
            }

            fn chunk_exists(&self, _chunk_id: &str) -> io::Result<bool> {
                Ok(false) // Nothing exists
            }
        }

        let temp_dir = tempdir().unwrap();
        let wal_path = temp_dir.path().join("checkpoint_recovery.wal");

        // Write WAL with checkpoint
        {
            let mut writer = WalWriter::create(&wal_path).unwrap();
            // Before checkpoint - these should NOT be replayed
            writer.append_add("t1", "c1", 1000, b"d1".to_vec()).unwrap();
            writer.append_add("t1", "c2", 2000, b"d2".to_vec()).unwrap();
            writer.append_checkpoint("t1", 3000).unwrap();
            // After checkpoint - these SHOULD be replayed
            writer.append_add("t1", "c3", 4000, b"d3".to_vec()).unwrap();
        }

        let reader = WalReader::open(&wal_path).unwrap();
        let mut handler = CountingHandler { add_count: 0 };

        let stats = replay(&reader, &mut handler).unwrap();

        // Only 1 add after checkpoint
        assert_eq!(stats.adds, 1);
        assert_eq!(handler.add_count, 1);
    }
}