simd_r_drive/
lib.rs

1// TODO: Integrate examples in main README
2
3//! # SIMD R Drive Storage Engine
4//!
5//! This crate provides an append-only, pinned storage engine optimized for
6//! high-performance applications. It is designed to offer:
7//! - **Zero-copy reads** using memory-mapped files.
8//! - **Append-only writes** ensuring immutability and data integrity.
9//! - **Efficient key lookups** via a custom hash index.
10//! - **Concurrent access** with atomic offsets and thread-safe locking.
11//!
12//! ## Features
13//! - **Append-Only Model**: Data can only be appended, never modified or deleted in place.
14//! - **Memory-Mapped Storage**: Efficient read performance with `memmap2`.
15//! - **SIMD Optimization**: Uses vectorized operations for fast memory copying.
16//! - **Fast Lookups**: Hash-based key index for quick retrieval.
17//! - **Crash Recovery**: Ensures that only valid data is loaded on restart.
18//!
19//! ## Example Usage
20//! ```rust
21//! use simd_r_drive::{DataStore, traits::{DataStoreReader, DataStoreWriter}};
22//! use std::path::PathBuf;
23//! use tempfile::tempdir;
24//!
25//! let temp_dir = tempdir().expect("Failed to create temp dir");
26//! let temp_path = temp_dir.path().join("test_storage.bin");
27//!
28//! // Open or create a new storage file
29//! let mut storage = DataStore::open(&PathBuf::from(temp_path)).unwrap();
30//!
31//! // Append some key-value entries
32//! storage.write(b"key1", b"value1").unwrap();
33//! storage.write(b"key2", b"value2").unwrap();
34//! storage.write(b"key3", b"value3").unwrap();
35//! storage.write(b"key4", b"value4").unwrap();
36//! storage.write(b"key5", b"value5").unwrap();
37//!
38//! // Retrieve some entries
39//! let entry_handle = storage.read(b"key1").unwrap().unwrap();
40//! assert_eq!(entry_handle.as_slice(), b"value1");
41//!
42//! let entry_handle = storage.read(b"key2").unwrap().unwrap();
43//! assert_eq!(entry_handle.as_slice(), b"value2");
44//!
45//! let entry_handle = storage.read(b"key3").unwrap().unwrap();
46//! assert_eq!(entry_handle.as_slice(), b"value3");
47//!
48//! let entry_handle = storage.read(b"key4").unwrap().unwrap();
49//! assert_eq!(entry_handle.as_slice(), b"value4");
50//!
51//! let entry_handle = storage.read(b"key5").unwrap().unwrap();
52//! assert_eq!(entry_handle.as_slice(), b"value5");
53//!
54//! // Overwrite an entry
55//! storage.write(b"key3", b"A new value").unwrap();
56//! let entry_handle = storage.read(b"key3").unwrap().unwrap();
57//! assert_eq!(entry_handle.as_slice(), b"A new value");
58//!
59//! // Delete an entry
60//! storage.delete(b"key3").unwrap();
61//! let entry_handle = storage.read(b"key3").unwrap();
62//! assert!(entry_handle.is_none());
63//! ```
64//!
65//! ## Streaming Example
66//! ```rust
67//! use simd_r_drive::{DataStore, EntryStream, traits::{DataStoreReader, DataStoreWriter}};
68//! use std::fs::File;
69//! use std::io::{Cursor, Read, Write};
70//! use std::path::PathBuf;
71//! use tempfile::tempdir;
72//!
73//! let temp_dir = tempdir().expect("Failed to create temp dir");
74//! let temp_path = temp_dir.path().join("test_storage_stream.bin");
75//!
76//! // Open or create a new storage file
77//! let mut storage = DataStore::open(&PathBuf::from(temp_path)).unwrap();
78//!
79//! // Example streaming data
80//! let stream_data = b"Streaming payload with large data";
81//! let mut cursor = Cursor::new(stream_data);
82//!
83//! // Write streaming data
84//! storage.write_stream(b"stream_key", &mut cursor).unwrap();
85//!
86//! // Read and validate streaming data using `EntryStream`
87//! let entry_handle = storage.read(b"stream_key").unwrap().unwrap(); // Get EntryHandle
88//! let mut retrieved_stream = EntryStream::from(entry_handle); // Convert to EntryStream
89//! let mut buffer = Vec::new();
90//!
91//! retrieved_stream.read_to_end(&mut buffer).unwrap(); // Read stream in chunks
92//! assert_eq!(buffer, stream_data);
93//!
94//! // Create a temporary file for testing
95//! let temp_path = "test_large_file.bin";
96//! let mut temp_file = File::create(temp_path).expect("Failed to create temp file");
97//! temp_file.write_all(b"Temporary file content").unwrap();
98//! temp_file.sync_all().unwrap(); // Ensure file is written
99//!
100//! // Open the file for streaming
101//! let mut file = File::open(temp_path).expect("File not found");
102//! storage.write_stream(b"file_stream_key", &mut file).unwrap();
103//!
104//! // Read back the streamed file using `EntryStream`
105//! let file_entry = storage.read(b"file_stream_key").unwrap().unwrap(); // Get EntryHandle
106//! let mut file_stream = EntryStream::from(file_entry); // Convert to EntryStream
107//! let mut file_buffer = Vec::new();
108//!
109//! file_stream.read_to_end(&mut file_buffer).unwrap();
110//!
111//! assert_eq!(file_buffer, b"Temporary file content");
112//!
113//! // Cleanup test file
114//! std::fs::remove_file(temp_path).unwrap();
115//! ```
116//!
117//! ## Performance Considerations
118//! - **Use memory-mapped reads** for best performance.
119//! - **Batch reads and writes** to reduce file I/O overhead.
120//! - **Avoid unnecessary locks** to maximize concurrency.
121//!
122//! ## Safety Notes
123//! - Memory-mapped files should not be resized while in use.
124//! - Ensure proper file synchronization after writes.
125//!
126//! ## License
127//! This project is licensed under the Apache-2.0 License.
128
129pub mod storage_engine;
130
131pub use storage_engine::digest::*;
132pub use storage_engine::*;
133
134pub mod utils;
135
136pub use storage_engine::NULL_BYTE;
simd_r_drive/lib.rs

simd_r_drive/
lib.rs
