1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
// I/O failure injection tests
// Tests error handling when I/O operations fail
// Critical for reliability: must handle I/O errors gracefully without data loss
use seerdb::{DBOptions, DB};
use std::fs::{self, OpenOptions};
use std::os::unix::fs::PermissionsExt;
use std::path::PathBuf;
use tempfile::TempDir;
#[test]
fn test_flush_failure_readonly_directory() {
// Test flush failure when data directory becomes read-only
let temp_dir = TempDir::new().unwrap();
let data_dir = PathBuf::from(temp_dir.path());
let db = DB::open(&data_dir).unwrap();
// Write data
for i in 0..100 {
db.put(format!("key_{:03}", i).as_bytes(), b"value")
.unwrap();
}
// Make directory read-only to simulate I/O failure
let mut perms = fs::metadata(&data_dir).unwrap().permissions();
perms.set_mode(0o444); // Read-only
fs::set_permissions(&data_dir, perms).unwrap();
// Flush should fail due to read-only directory
let result = db.flush();
// Restore permissions for cleanup
let mut perms = fs::metadata(&data_dir).unwrap().permissions();
perms.set_mode(0o755);
fs::set_permissions(&data_dir, perms).unwrap();
// Flush should have failed
assert!(
result.is_err(),
"Flush should fail with read-only directory"
);
// Verify data is still in memtable (not lost)
for i in 0..100 {
assert!(
db.get(format!("key_{:03}", i).as_bytes())
.unwrap()
.is_some(),
"Data should remain in memtable after flush failure"
);
}
}
#[test]
fn test_recovery_after_flush_failure() {
// Test that DB can recover after a flush failure
let temp_dir = TempDir::new().unwrap();
let data_dir = PathBuf::from(temp_dir.path());
// Write data
{
let db = DB::open(&data_dir).unwrap();
for i in 0..50 {
db.put(format!("key_{:03}", i).as_bytes(), b"value")
.unwrap();
}
// Make directory read-only
let mut perms = fs::metadata(&data_dir).unwrap().permissions();
perms.set_mode(0o444);
fs::set_permissions(&data_dir, perms).unwrap();
// Try to flush (will fail)
let _ = db.flush();
// Restore permissions
let mut perms = fs::metadata(&data_dir).unwrap().permissions();
perms.set_mode(0o755);
fs::set_permissions(&data_dir, perms).unwrap();
// DB goes out of scope - should write WAL
}
// Reopen - data should be recovered from WAL
{
let db = DB::open(&data_dir).unwrap();
for i in 0..50 {
assert!(
db.get(format!("key_{:03}", i).as_bytes())
.unwrap()
.is_some(),
"Data should be recovered from WAL after flush failure"
);
}
}
}
#[test]
fn test_corrupted_sstable_skipped() {
// Test that DB can open even if one SSTable is corrupted
let temp_dir = TempDir::new().unwrap();
let data_dir = PathBuf::from(temp_dir.path());
// Write and flush multiple SSTables
{
let db = DBOptions::default()
.memtable_capacity(1024) // Small to force flushes
.open(&data_dir)
.unwrap();
// Write first batch
for i in 0..100 {
db.put(format!("batch1_{:03}", i).as_bytes(), &vec![b'a'; 100])
.unwrap();
}
db.flush().unwrap();
// Write second batch
for i in 0..100 {
db.put(format!("batch2_{:03}", i).as_bytes(), &vec![b'b'; 100])
.unwrap();
}
db.flush().unwrap();
}
// Corrupt the first SSTable
let sstable_path = data_dir.join("L0_000000.sst");
if sstable_path.exists() {
fs::remove_file(&sstable_path).unwrap();
// Create empty file (simulates corruption)
fs::File::create(&sstable_path).unwrap();
}
// Reopen - should handle corrupted SSTable
{
let result = DB::open(&data_dir);
match result {
Ok(db) => {
// DB opened despite corruption
// Second batch should still be readable
let readable = (0..100)
.filter(|i| {
db.get(format!("batch2_{:03}", i).as_bytes())
.unwrap()
.is_some()
})
.count();
// Should be able to read uncorrupted data
assert!(readable > 0, "Should read data from uncorrupted SSTables");
}
Err(_) => {
// Also acceptable - strict corruption detection
}
}
}
}
#[test]
fn test_partial_wal_recovery() {
// Test recovery when WAL is partially written
let temp_dir = TempDir::new().unwrap();
let data_dir = PathBuf::from(temp_dir.path());
// Write data without flushing
{
let db = DB::open(&data_dir).unwrap();
for i in 0..100 {
db.put(format!("key_{:03}", i).as_bytes(), b"value")
.unwrap();
}
// Don't flush - data only in WAL
}
// Truncate WAL to simulate partial write
let wal_path = data_dir.join("wal.log");
if wal_path.exists() {
let metadata = fs::metadata(&wal_path).unwrap();
let size = metadata.len();
let file = OpenOptions::new().write(true).open(&wal_path).unwrap();
file.set_len(size / 2).unwrap(); // Cut in half
}
// Reopen - should recover partial data
{
let result = DB::open(&data_dir);
match result {
Ok(db) => {
// Count recovered keys
let recovered = (0..100)
.filter(|i| {
db.get(format!("key_{:03}", i).as_bytes())
.unwrap()
.is_some()
})
.count();
// Should recover at least some data before truncation point
// But not all 100 keys
assert!(
recovered < 100,
"Should not recover all keys from truncated WAL"
);
}
Err(_) => {
// Also acceptable - may reject truncated WAL entirely
}
}
}
}
#[test]
fn test_write_data_despite_wal_issues() {
// Test that we can still write data even if WAL has issues
// (though durability may be compromised)
let temp_dir = TempDir::new().unwrap();
let data_dir = PathBuf::from(temp_dir.path());
let db = DB::open(&data_dir).unwrap();
// Normal writes should work
for i in 0..50 {
db.put(format!("key_{:03}", i).as_bytes(), b"value")
.unwrap();
}
// Verify data is readable
for i in 0..50 {
assert!(db
.get(format!("key_{:03}", i).as_bytes())
.unwrap()
.is_some());
}
}
#[test]
fn test_operations_after_failed_flush() {
// Test that DB can recover and continue working after a flush failure
let temp_dir = TempDir::new().unwrap();
let data_dir = PathBuf::from(temp_dir.path());
let db = DB::open(&data_dir).unwrap();
// Write initial data
for i in 0..50 {
db.put(format!("key_{:03}", i).as_bytes(), b"value")
.unwrap();
}
// Make directory read-only to cause flush failure
let mut perms = fs::metadata(&data_dir).unwrap().permissions();
perms.set_mode(0o444);
fs::set_permissions(&data_dir, perms).unwrap();
// Try flush (will fail)
let flush_result = db.flush();
assert!(
flush_result.is_err(),
"Flush should fail with read-only dir"
);
// Restore permissions IMMEDIATELY for subsequent operations
let mut perms = fs::metadata(&data_dir).unwrap().permissions();
perms.set_mode(0o755);
fs::set_permissions(&data_dir, perms).unwrap();
// Old data should still be readable (in immutable_memtable)
let old_data_readable = (0..50)
.filter(|i| {
db.get(format!("key_{:03}", i).as_bytes())
.unwrap()
.is_some()
})
.count();
assert!(
old_data_readable > 0,
"Old data should be readable after flush failure (in immutable_memtable)"
);
// DB should accept new writes
for i in 50..100 {
db.put(format!("key_{:03}", i).as_bytes(), b"value")
.unwrap();
}
// New data should be readable
let new_data_readable = (50..100)
.filter(|i| {
db.get(format!("key_{:03}", i).as_bytes())
.unwrap()
.is_some()
})
.count();
assert!(
new_data_readable > 0,
"New data should be writable and readable after flush failure"
);
// Successful flush should work now
db.flush().unwrap();
// All data should be readable after successful flush
for i in 0..100 {
assert!(
db.get(format!("key_{:03}", i).as_bytes())
.unwrap()
.is_some(),
"All data should be readable after successful flush"
);
}
}
#[test]
fn test_missing_sstable_file() {
// Test behavior when an SSTable file is deleted while DB is closed
let temp_dir = TempDir::new().unwrap();
let data_dir = PathBuf::from(temp_dir.path());
// Create and flush data
{
let db = DB::open(&data_dir).unwrap();
for i in 0..100 {
db.put(format!("key_{:03}", i).as_bytes(), b"value")
.unwrap();
}
db.flush().unwrap();
}
// Delete ALL SSTable files (there may be multiple)
for entry in fs::read_dir(&data_dir).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.extension().and_then(|s| s.to_str()) == Some("sst") {
fs::remove_file(&path).unwrap();
}
}
// Also delete WAL to truly test missing files scenario
let wal_path = data_dir.join("wal.log");
if wal_path.exists() {
fs::remove_file(&wal_path).unwrap();
}
// Reopen - should handle missing file
{
let result = DB::open(&data_dir);
match result {
Ok(db) => {
// Opened but data should be lost (no SSTable, no WAL)
let found = (0..100)
.filter(|i| {
db.get(format!("key_{:03}", i).as_bytes())
.unwrap()
.is_some()
})
.count();
assert!(
found == 0,
"Data should be lost if both SSTable and WAL are missing"
);
// DB should still be usable for new writes
db.put(b"new_key", b"new_value").unwrap();
assert_eq!(db.get(b"new_key").unwrap().unwrap().as_ref(), b"new_value");
}
Err(_) => {
// Also acceptable - strict file integrity check
}
}
}
}
#[test]
fn test_error_propagation() {
// Test that I/O errors are propagated as Err, not panics
let temp_dir = TempDir::new().unwrap();
let data_dir = PathBuf::from(temp_dir.path());
let db = DB::open(&data_dir).unwrap();
db.put(b"key", b"value").unwrap();
// Make directory read-only
let mut perms = fs::metadata(&data_dir).unwrap().permissions();
perms.set_mode(0o444);
fs::set_permissions(&data_dir, perms).unwrap();
// Operations should return Err, not panic
let flush_result = db.flush();
assert!(flush_result.is_err(), "Should return Err on I/O failure");
// Restore permissions
let mut perms = fs::metadata(&data_dir).unwrap().permissions();
perms.set_mode(0o755);
fs::set_permissions(&data_dir, perms).unwrap();
// DB should still be usable after error
assert_eq!(db.get(b"key").unwrap().unwrap().as_ref(), b"value");
}