heed/env.rs
1use std::any::TypeId;
2use std::cmp::Ordering;
3use std::collections::hash_map::{Entry, HashMap};
4use std::ffi::{c_void, CString};
5#[cfg(target_os = "linux")]
6use std::fs::OpenOptions;
7use std::fs::{File, Metadata};
8use std::io::ErrorKind::NotFound;
9#[cfg(target_os = "linux")]
10use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
11#[cfg(unix)]
12use std::os::unix::{
13 ffi::OsStrExt,
14 io::{AsRawFd, BorrowedFd, RawFd},
15};
16use std::panic::catch_unwind;
17use std::path::{Path, PathBuf};
18use std::process::abort;
19use std::sync::{Arc, RwLock};
20use std::time::Duration;
21#[cfg(windows)]
22use std::{
23 ffi::OsStr,
24 os::windows::io::{AsRawHandle, BorrowedHandle, RawHandle},
25};
26use std::{fmt, io, mem, ptr};
27
28use heed_traits::{Comparator, LexicographicComparator};
29use once_cell::sync::Lazy;
30use synchronoise::event::SignalEvent;
31
32use crate::cursor::MoveOperation;
33use crate::database::DatabaseOpenOptions;
34use crate::mdb::error::mdb_result;
35use crate::mdb::ffi;
36use crate::mdb::lmdb_flags::AllDatabaseFlags;
37use crate::{
38 Database, EnvFlags, Error, Result, RoCursor, RoTxn, RwTxn, Unspecified, WriteAdmission,
39};
40
41/// The list of opened environments, the value is an optional environment, it is None
42/// when someone asks to close the environment, closing is a two-phase step, to make sure
43/// noone tries to open the same environment between these two phases.
44///
45/// Trying to open a None marked environment returns an error to the user trying to open it.
46static OPENED_ENV: Lazy<RwLock<HashMap<PathBuf, EnvEntry>>> = Lazy::new(RwLock::default);
47
48struct EnvEntry {
49 env: Option<Env>,
50 signal_event: Arc<SignalEvent>,
51 options: EnvOpenOptions,
52}
53
54/// Caller-pinned identity of one LMDB file.
55#[derive(Clone, Copy, Debug, PartialEq, Eq)]
56pub struct PinnedLmdbFileIdentity {
57 /// Device number reported by `stat`.
58 pub device: u64,
59 /// Inode number reported by `stat`.
60 pub inode: u64,
61}
62
63/// Caller-pinned identities for both files in an LMDB environment.
64///
65/// The authority-establishing caller supplies these identities. Capturing
66/// them inside `open` would leave a pathname-swap window between validation
67/// and the LMDB open.
68#[derive(Clone, Copy, Debug, PartialEq, Eq)]
69#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
70pub struct PinnedLmdbIdentity {
71 /// Authorized `data.mdb` identity.
72 pub data: PinnedLmdbFileIdentity,
73 /// Authorized `lock.mdb` identity.
74 pub lock: PinnedLmdbFileIdentity,
75}
76
77impl PinnedLmdbIdentity {
78 #[cfg(target_os = "linux")]
79 fn validate_before_open(&self, path: &Path) -> io::Result<()> {
80 if pinned_file_identity(&path.join("data.mdb"))? != self.data
81 || pinned_file_identity(&path.join("lock.mdb"))? != self.lock
82 {
83 return Err(io::Error::other(
84 "LMDB data/lock entry differs from the caller-pinned identity",
85 ));
86 }
87 Ok(())
88 }
89
90 #[cfg(not(target_os = "linux"))]
91 fn validate_before_open(&self, _path: &Path) -> io::Result<()> {
92 Err(io::Error::new(
93 io::ErrorKind::Unsupported,
94 "pinned LMDB paths are supported only on Linux",
95 ))
96 }
97
98 #[cfg(target_os = "linux")]
99 unsafe fn validate_after_open(&self, env: *mut ffi::MDB_env, path: &Path) -> io::Result<()> {
100 if pinned_file_identity(&path.join("data.mdb"))? != self.data
101 || pinned_file_identity(&path.join("lock.mdb"))? != self.lock
102 {
103 return Err(io::Error::other(
104 "pinned LMDB data/lock entry changed during open",
105 ));
106 }
107 let mut data_fd = -1;
108 let status = unsafe { ffi::mdb_env_get_fd(env, &mut data_fd) };
109 if status != 0 {
110 return Err(io::Error::from_raw_os_error(status));
111 }
112 if pinned_fd_identity(data_fd)? != self.data {
113 return Err(io::Error::other(
114 "LMDB opened a different data.mdb than the pinned entry",
115 ));
116 }
117 let mut lock_fd = -1;
118 let status = unsafe { ffi::mdb_env_get_lockfd(env, &mut lock_fd) };
119 if status != 0 {
120 return Err(io::Error::from_raw_os_error(status));
121 }
122 if pinned_fd_identity(lock_fd)? != self.lock {
123 return Err(io::Error::other(
124 "LMDB opened a different lock.mdb than the pinned entry",
125 ));
126 }
127 Ok(())
128 }
129
130 #[cfg(not(target_os = "linux"))]
131 unsafe fn validate_after_open(&self, _env: *mut ffi::MDB_env, _path: &Path) -> io::Result<()> {
132 Err(io::Error::new(
133 io::ErrorKind::Unsupported,
134 "pinned LMDB paths are supported only on Linux",
135 ))
136 }
137}
138
139#[cfg(target_os = "linux")]
140fn pinned_file_identity(path: &Path) -> io::Result<PinnedLmdbFileIdentity> {
141 let mut options = OpenOptions::new();
142 options
143 .read(true)
144 .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK);
145 let file = options.open(path)?;
146 let metadata = file.metadata()?;
147 if !metadata.is_file() {
148 return Err(io::Error::new(
149 io::ErrorKind::InvalidData,
150 "pinned LMDB entry is not a regular file",
151 ));
152 }
153 Ok(PinnedLmdbFileIdentity {
154 device: metadata.dev(),
155 inode: metadata.ino(),
156 })
157}
158
159#[cfg(target_os = "linux")]
160fn pinned_fd_identity(fd: libc::c_int) -> io::Result<PinnedLmdbFileIdentity> {
161 let mut stat = mem::MaybeUninit::<libc::stat>::zeroed();
162 if unsafe { libc::fstat(fd, stat.as_mut_ptr()) } != 0 {
163 return Err(io::Error::last_os_error());
164 }
165 let stat = unsafe { stat.assume_init() };
166 Ok(PinnedLmdbFileIdentity {
167 device: stat.st_dev as u64,
168 inode: stat.st_ino as u64,
169 })
170}
171
172// Thanks to the mozilla/rkv project
173// Workaround the UNC path on Windows, see https://github.com/rust-lang/rust/issues/42869.
174// Otherwise, `Env::from_env()` will panic with error_no(123).
175#[cfg(not(windows))]
176fn canonicalize_path(path: &Path) -> io::Result<PathBuf> {
177 path.canonicalize()
178}
179
180#[cfg(windows)]
181fn canonicalize_path(path: &Path) -> io::Result<PathBuf> {
182 let canonical = path.canonicalize()?;
183 let url = url::Url::from_file_path(&canonical)
184 .map_err(|_e| io::Error::new(io::ErrorKind::Other, "URL passing error"))?;
185 url.to_file_path()
186 .map_err(|_e| io::Error::new(io::ErrorKind::Other, "path canonicalization error"))
187}
188
189#[cfg(windows)]
190/// Adding a 'missing' trait from windows OsStrExt
191trait OsStrExtLmdb {
192 fn as_bytes(&self) -> &[u8];
193}
194#[cfg(windows)]
195impl OsStrExtLmdb for OsStr {
196 fn as_bytes(&self) -> &[u8] {
197 &self.to_str().unwrap().as_bytes()
198 }
199}
200
201#[cfg(unix)]
202fn get_file_fd(file: &File) -> RawFd {
203 file.as_raw_fd()
204}
205
206#[cfg(windows)]
207fn get_file_fd(file: &File) -> RawHandle {
208 file.as_raw_handle()
209}
210
211#[cfg(unix)]
212/// Get metadata from a file descriptor.
213unsafe fn metadata_from_fd(raw_fd: RawFd) -> io::Result<Metadata> {
214 let fd = BorrowedFd::borrow_raw(raw_fd);
215 let owned = fd.try_clone_to_owned()?;
216 File::from(owned).metadata()
217}
218
219#[cfg(windows)]
220/// Get metadata from a file descriptor.
221unsafe fn metadata_from_fd(raw_fd: RawHandle) -> io::Result<Metadata> {
222 let fd = BorrowedHandle::borrow_raw(raw_fd);
223 let owned = fd.try_clone_to_owned()?;
224 File::from(owned).metadata()
225}
226
227/// Options and flags which can be used to configure how an environment is opened.
228#[derive(Clone, Debug, PartialEq)]
229#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
230pub struct EnvOpenOptions {
231 map_size: Option<usize>,
232 max_readers: Option<u32>,
233 max_dbs: Option<u32>,
234 flags: EnvFlags,
235 #[cfg_attr(
236 feature = "serde",
237 serde(default, skip_serializing_if = "Option::is_none")
238 )]
239 write_admission: Option<WriteAdmission>,
240}
241
242impl Default for EnvOpenOptions {
243 fn default() -> Self {
244 Self::new()
245 }
246}
247
248impl EnvOpenOptions {
249 /// Creates a blank new set of options ready for configuration.
250 pub fn new() -> EnvOpenOptions {
251 EnvOpenOptions {
252 map_size: None,
253 max_readers: None,
254 max_dbs: None,
255 flags: EnvFlags::empty(),
256 write_admission: None,
257 }
258 }
259
260 /// Install admission before environment initialization and every physical
261 /// data write or lock-file extension. Clone one handle for compatible opens.
262 pub fn write_admission(&mut self, admission: WriteAdmission) -> &mut Self {
263 self.write_admission = Some(admission);
264 self
265 }
266
267 /// Set the size of the memory map to use for this environment.
268 pub fn map_size(&mut self, size: usize) -> &mut Self {
269 self.map_size = Some(size);
270 self
271 }
272
273 /// Set the maximum number of threads/reader slots for the environment.
274 pub fn max_readers(&mut self, readers: u32) -> &mut Self {
275 self.max_readers = Some(readers);
276 self
277 }
278
279 /// Set the maximum number of named databases for the environment.
280 pub fn max_dbs(&mut self, dbs: u32) -> &mut Self {
281 self.max_dbs = Some(dbs);
282 self
283 }
284
285 /// Set one or more [LMDB flags](http://www.lmdb.tech/doc/group__mdb__env.html).
286 /// ```
287 /// use std::fs;
288 /// use std::path::Path;
289 /// use heed::{EnvOpenOptions, Database, EnvFlags};
290 /// use heed::types::*;
291 ///
292 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
293 /// fs::create_dir_all(Path::new("target").join("database.mdb"))?;
294 /// let mut env_builder = EnvOpenOptions::new();
295 /// unsafe { env_builder.flags(EnvFlags::NO_TLS | EnvFlags::NO_META_SYNC); }
296 /// let dir = tempfile::tempdir().unwrap();
297 /// let env = unsafe { env_builder.open(dir.path())? };
298 ///
299 /// // we will open the default unnamed database
300 /// let mut wtxn = env.write_txn()?;
301 /// let db: Database<Str, U32<byteorder::NativeEndian>> = env.create_database(&mut wtxn, None)?;
302 ///
303 /// // opening a write transaction
304 /// db.put(&mut wtxn, "seven", &7)?;
305 /// db.put(&mut wtxn, "zero", &0)?;
306 /// db.put(&mut wtxn, "five", &5)?;
307 /// db.put(&mut wtxn, "three", &3)?;
308 /// wtxn.commit()?;
309 ///
310 /// // force the OS to flush the buffers (see Flag::NoSync and Flag::NoMetaSync).
311 /// env.force_sync();
312 ///
313 /// // opening a read transaction
314 /// // to check if those values are now available
315 /// let mut rtxn = env.read_txn()?;
316 ///
317 /// let ret = db.get(&rtxn, "zero")?;
318 /// assert_eq!(ret, Some(0));
319 ///
320 /// let ret = db.get(&rtxn, "five")?;
321 /// assert_eq!(ret, Some(5));
322 /// # Ok(()) }
323 /// ```
324 ///
325 /// # Safety
326 ///
327 /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`.
328 pub unsafe fn flags(&mut self, flags: EnvFlags) -> &mut Self {
329 self.flags |= flags;
330 self
331 }
332
333 /// Open an environment that will be located at the specified path.
334 ///
335 /// # Safety
336 /// LMDB is backed by a memory map [^1] which comes with some safety precautions.
337 ///
338 /// Memory map constructors are marked `unsafe` because of the potential
339 /// for Undefined Behavior (UB) using the map if the underlying file is
340 /// subsequently modified, in or out of process.
341 ///
342 /// LMDB itself has a locking system that solves this problem,
343 /// but it will not save you from making mistakes yourself.
344 ///
345 /// These are some things to take note of:
346 ///
347 /// - Avoid long-lived transactions, they will cause the database to grow quickly [^2]
348 /// - Avoid aborting your process with an active transaction [^3]
349 /// - Do not use LMDB on remote filesystems, even between processes on the same host [^4]
350 /// - You must manage concurrent accesses yourself if using [`EnvFlags::NO_LOCK`] [^5]
351 /// - Anything that causes LMDB's lock file to be broken will cause synchronization issues and may introduce UB [^6]
352 ///
353 /// `heed` itself upholds some safety invariants, including but not limited to:
354 /// - Calling [`EnvOpenOptions::open`] twice in the same process, at the same time is OK [^7]
355 ///
356 /// For more details, it is highly recommended to read LMDB's official documentation. [^8]
357 ///
358 /// [^1]: <https://en.wikipedia.org/wiki/Memory_map>
359 ///
360 /// [^2]: <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L107-L114>
361 ///
362 /// [^3]: <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L118-L121>
363 ///
364 /// [^4]: <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L129>
365 ///
366 /// [^5]: <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L129>
367 ///
368 /// [^6]: <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L49-L52>
369 ///
370 /// [^7]: <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L102-L105>
371 ///
372 /// [^8]: <http://www.lmdb.tech/doc/index.html>
373 pub unsafe fn open<P: AsRef<Path>>(&self, path: P) -> Result<Env> {
374 let path = match canonicalize_path(path.as_ref()) {
375 Err(err) => {
376 if err.kind() == NotFound && self.flags.contains(EnvFlags::NO_SUB_DIR) {
377 let path = path.as_ref();
378 match path.parent().zip(path.file_name()) {
379 Some((dir, file_name)) => canonicalize_path(dir)?.join(file_name),
380 None => return Err(err.into()),
381 }
382 } else {
383 return Err(err.into());
384 }
385 }
386 Ok(path) => path,
387 };
388 unsafe { self.open_resolved_path(path, None) }
389 }
390
391 /// Opens an environment through an already-pinned absolute path without
392 /// canonicalizing it first.
393 ///
394 /// This is intended for Linux `/proc/self/fd/N` directory handles. The
395 /// ordinary [`Self::open`] canonicalization would turn such a handle back
396 /// into a mutable pathname before LMDB opens its files, defeating the
397 /// caller's retained directory descriptor.
398 ///
399 /// # Safety
400 ///
401 /// In addition to [`Self::open`]'s requirements, `path` must be an
402 /// absolute, process-owned, retained path whose identity cannot be
403 /// redirected for the lifetime of the returned environment.
404 pub unsafe fn open_from_pinned_path<P: AsRef<Path>>(
405 &self,
406 path: P,
407 pinned: PinnedLmdbIdentity,
408 ) -> Result<Env> {
409 let path = path.as_ref();
410 if !path.is_absolute() {
411 return Err(Error::Io(io::Error::new(
412 io::ErrorKind::InvalidInput,
413 "pinned LMDB path must be absolute",
414 )));
415 }
416 pinned.validate_before_open(path)?;
417 unsafe { self.open_resolved_path(path.to_path_buf(), Some(pinned)) }
418 }
419
420 unsafe fn open_resolved_path(
421 &self,
422 path: PathBuf,
423 pinned: Option<PinnedLmdbIdentity>,
424 ) -> Result<Env> {
425 #[cfg(not(unix))]
426 if self.write_admission.is_some() {
427 return Err(Error::Io(io::Error::new(
428 io::ErrorKind::Unsupported,
429 "LMDB write admission requires Unix file descriptors",
430 )));
431 }
432 let mut lock = OPENED_ENV.write().unwrap();
433 match lock.entry(path) {
434 Entry::Occupied(entry) => {
435 let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?;
436 let options = entry.get().options.clone();
437 if &options == self {
438 Ok(env)
439 } else {
440 Err(Error::BadOpenOptions { env, options })
441 }
442 }
443 Entry::Vacant(entry) => {
444 let path = entry.key();
445 let path_str = CString::new(path.as_os_str().as_bytes()).unwrap();
446
447 unsafe {
448 let mut env: *mut ffi::MDB_env = ptr::null_mut();
449 mdb_result(ffi::mdb_env_create(&mut env))?;
450
451 #[cfg(unix)]
452 if let Some(admission) = &self.write_admission {
453 if let Err(error) = mdb_result(ffi::mdb_env_set_write_admission(
454 env,
455 Some(crate::write_admission::admit_write),
456 admission.context(),
457 )) {
458 ffi::mdb_env_close(env);
459 return Err(error.into());
460 }
461 }
462
463 if let Some(pinned) = pinned {
464 mdb_result(ffi::mdb_env_set_expected_file_identity(
465 env,
466 pinned.data.device,
467 pinned.data.inode,
468 pinned.lock.device,
469 pinned.lock.inode,
470 ))?;
471 }
472
473 if let Some(size) = self.map_size {
474 if size % page_size::get() != 0 {
475 let msg = format!(
476 "map size ({}) must be a multiple of the system page size ({})",
477 size,
478 page_size::get()
479 );
480 return Err(Error::Io(io::Error::new(
481 io::ErrorKind::InvalidInput,
482 msg,
483 )));
484 }
485 mdb_result(ffi::mdb_env_set_mapsize(env, size))?;
486 }
487
488 if let Some(readers) = self.max_readers {
489 mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?;
490 }
491
492 if let Some(dbs) = self.max_dbs {
493 mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?;
494 }
495
496 // When the `read-txn-no-tls` feature is enabled, we must force LMDB
497 // to avoid using the thread local storage, this way we allow users
498 // to use references of RoTxn between threads safely.
499 let flags = if cfg!(feature = "read-txn-no-tls") {
500 self.flags | EnvFlags::NO_TLS
501 } else {
502 self.flags
503 };
504
505 let result = mdb_result(ffi::mdb_env_open(
506 env,
507 path_str.as_ptr(),
508 flags.bits(),
509 0o600,
510 ));
511
512 match result {
513 Ok(()) => {
514 if let Some(pinned) = pinned {
515 if let Err(error) = pinned.validate_after_open(env, path) {
516 ffi::mdb_env_close(env);
517 return Err(Error::Io(error));
518 }
519 }
520 let signal_event = Arc::new(SignalEvent::manual(false));
521 let inner = EnvInner {
522 env,
523 path: path.clone(),
524 _write_admission: self.write_admission.clone(),
525 };
526 let env = Env(Arc::new(inner));
527 let cache_entry = EnvEntry {
528 env: Some(env.clone()),
529 options: self.clone(),
530 signal_event,
531 };
532 entry.insert(cache_entry);
533 Ok(env)
534 }
535 Err(e) => {
536 ffi::mdb_env_close(env);
537 Err(e.into())
538 }
539 }
540 }
541 }
542 }
543 }
544}
545
546/// Returns a struct that allows to wait for the effective closing of an environment.
547pub fn env_closing_event<P: AsRef<Path>>(path: P) -> Option<EnvClosingEvent> {
548 let lock = OPENED_ENV.read().unwrap();
549 lock.get(path.as_ref())
550 .map(|e| EnvClosingEvent(e.signal_event.clone()))
551}
552
553/// An environment handle constructed by using [`EnvOpenOptions`].
554#[derive(Clone)]
555pub struct Env(Arc<EnvInner>);
556
557impl fmt::Debug for Env {
558 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
559 let EnvInner { path, .. } = self.0.as_ref();
560 f.debug_struct("Env")
561 .field("path", &path.display())
562 .finish_non_exhaustive()
563 }
564}
565
566struct EnvInner {
567 env: *mut ffi::MDB_env,
568 path: PathBuf,
569 _write_admission: Option<WriteAdmission>,
570}
571
572unsafe impl Send for EnvInner {}
573
574unsafe impl Sync for EnvInner {}
575
576impl Drop for EnvInner {
577 fn drop(&mut self) {
578 let mut lock = OPENED_ENV.write().unwrap();
579
580 match lock.remove(&self.path) {
581 None => panic!("It seems another env closed this env before"),
582 Some(EnvEntry { signal_event, .. }) => {
583 unsafe {
584 ffi::mdb_env_close(self.env);
585 }
586 // We signal to all the waiters that the env is closed now.
587 signal_event.signal();
588 }
589 }
590 }
591}
592
593/// A helper function that transforms the LMDB types into Rust types (`MDB_val` into slices)
594/// and vice versa, the Rust types into C types (`Ordering` into an integer).
595///
596/// # Safety
597///
598/// `a` and `b` should both properly aligned, valid for reads and should point to a valid
599/// [`MDB_val`][ffi::MDB_val]. An [`MDB_val`][ffi::MDB_val] (consists of a pointer and size) is
600/// valid when its pointer (`mv_data`) is valid for reads of `mv_size` bytes and is not null.
601unsafe extern "C" fn custom_key_cmp_wrapper<C: Comparator>(
602 a: *const ffi::MDB_val,
603 b: *const ffi::MDB_val,
604) -> i32 {
605 let a = unsafe { ffi::from_val(*a) };
606 let b = unsafe { ffi::from_val(*b) };
607 match catch_unwind(|| C::compare(a, b)) {
608 Ok(Ordering::Less) => -1,
609 Ok(Ordering::Equal) => 0,
610 Ok(Ordering::Greater) => 1,
611 Err(_) => abort(),
612 }
613}
614
615/// A representation of LMDB's default comparator behavior.
616///
617/// This enum is used to indicate the absence of a custom comparator for an LMDB
618/// database instance. When a [`Database`] is created or opened with
619/// [`DefaultComparator`], it signifies that the comparator should not be explicitly
620/// set via [`ffi::mdb_set_compare`]. Consequently, the database
621/// instance utilizes LMDB's built-in default comparator, which inherently performs
622/// lexicographic comparison of keys.
623///
624/// This comparator's lexicographic implementation is employed in scenarios involving
625/// prefix iterators. Specifically, methods other than [`Comparator::compare`] are utilized
626/// to determine the lexicographic successors and predecessors of byte sequences, which
627/// is essential for these iterators' operation.
628///
629/// When a custom comparator is provided, the wrapper is responsible for setting
630/// it with the [`ffi::mdb_set_compare`] function, which overrides the default comparison
631/// behavior of LMDB with the user-defined logic.
632pub enum DefaultComparator {}
633
634impl LexicographicComparator for DefaultComparator {
635 #[inline]
636 fn compare_elem(a: u8, b: u8) -> Ordering {
637 a.cmp(&b)
638 }
639
640 #[inline]
641 fn successor(elem: u8) -> Option<u8> {
642 match elem {
643 u8::MAX => None,
644 elem => Some(elem + 1),
645 }
646 }
647
648 #[inline]
649 fn predecessor(elem: u8) -> Option<u8> {
650 match elem {
651 u8::MIN => None,
652 elem => Some(elem - 1),
653 }
654 }
655
656 #[inline]
657 fn max_elem() -> u8 {
658 u8::MAX
659 }
660
661 #[inline]
662 fn min_elem() -> u8 {
663 u8::MIN
664 }
665}
666
667/// Whether to perform compaction while copying an environment.
668#[derive(Debug, Copy, Clone)]
669pub enum CompactionOption {
670 /// Omit free pages and sequentially renumber all pages in output.
671 ///
672 /// This option consumes more CPU and runs more slowly than the default.
673 /// Currently it fails if the environment has suffered a page leak.
674 Enabled,
675
676 /// Copy everything without taking any special action about free pages.
677 Disabled,
678}
679
680/// Whether to enable or disable flags in [`Env::set_flags`].
681#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
682pub enum FlagSetMode {
683 /// Enable the flags.
684 Enable,
685 /// Disable the flags.
686 Disable,
687}
688
689impl FlagSetMode {
690 /// Convert the enum into the `i32` required by LMDB.
691 /// "A non-zero value sets the flags, zero clears them."
692 /// <http://www.lmdb.tech/doc/group__mdb.html#ga83f66cf02bfd42119451e9468dc58445>
693 fn as_mdb_env_set_flags_input(self) -> i32 {
694 match self {
695 Self::Enable => 1,
696 Self::Disable => 0,
697 }
698 }
699}
700
701impl Env {
702 pub(crate) fn env_mut_ptr(&self) -> *mut ffi::MDB_env {
703 self.0.env
704 }
705
706 /// The size of the data file on disk.
707 ///
708 /// # Example
709 ///
710 /// ```
711 /// use heed::EnvOpenOptions;
712 ///
713 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
714 /// let dir = tempfile::tempdir()?;
715 /// let size_in_bytes = 1024 * 1024;
716 /// let env = unsafe { EnvOpenOptions::new().map_size(size_in_bytes).open(dir.path())? };
717 ///
718 /// let actual_size = env.real_disk_size()? as usize;
719 /// assert!(actual_size < size_in_bytes);
720 /// # Ok(()) }
721 /// ```
722 pub fn real_disk_size(&self) -> Result<u64> {
723 let mut fd = mem::MaybeUninit::uninit();
724 unsafe { mdb_result(ffi::mdb_env_get_fd(self.env_mut_ptr(), fd.as_mut_ptr()))? };
725 let fd = unsafe { fd.assume_init() };
726 let metadata = unsafe { metadata_from_fd(fd)? };
727 Ok(metadata.len())
728 }
729
730 /// Return the raw flags the environment was opened with.
731 ///
732 /// Returns `None` if the environment flags are different from the [`EnvFlags`] set.
733 pub fn flags(&self) -> Result<Option<EnvFlags>> {
734 self.get_flags().map(EnvFlags::from_bits)
735 }
736
737 /// Enable or disable the environment's currently active [`EnvFlags`].
738 ///
739 /// ```
740 /// use std::fs;
741 /// use std::path::Path;
742 /// use heed::{EnvOpenOptions, Database, EnvFlags, FlagSetMode};
743 /// use heed::types::*;
744 ///
745 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
746 /// fs::create_dir_all(Path::new("target").join("database.mdb"))?;
747 /// let mut env_builder = EnvOpenOptions::new();
748 /// let dir = tempfile::tempdir().unwrap();
749 /// let env = unsafe { env_builder.open(dir.path())? };
750 ///
751 /// // Preserve the flags enabled by default.
752 /// let initial_flags = env.get_flags()?;
753 /// assert_eq!(initial_flags & EnvFlags::NO_SYNC.bits(), 0);
754 ///
755 /// // Enable a flag after opening.
756 /// unsafe { env.set_flags(EnvFlags::NO_SYNC, FlagSetMode::Enable).unwrap(); }
757 /// assert_eq!(env.get_flags()?, initial_flags | EnvFlags::NO_SYNC.bits());
758 ///
759 /// // Disable a flag after opening.
760 /// unsafe { env.set_flags(EnvFlags::NO_SYNC, FlagSetMode::Disable).unwrap(); }
761 /// assert_eq!(env.get_flags()?, initial_flags);
762 /// # Ok(()) }
763 /// ```
764 ///
765 /// # Safety
766 ///
767 /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`.
768 ///
769 /// LMDB also requires that only 1 thread calls this function at any given moment.
770 /// Neither `heed` or LMDB check for this condition, so the caller must ensure it explicitly.
771 pub unsafe fn set_flags(&self, flags: EnvFlags, mode: FlagSetMode) -> Result<()> {
772 // safety: caller must ensure no other thread is calling this function.
773 // <http://www.lmdb.tech/doc/group__mdb.html#ga83f66cf02bfd42119451e9468dc58445>
774 mdb_result(unsafe {
775 ffi::mdb_env_set_flags(
776 self.env_mut_ptr(),
777 flags.bits(),
778 mode.as_mdb_env_set_flags_input(),
779 )
780 })
781 .map_err(Into::into)
782 }
783
784 /// Return the raw flags the environment is currently set with.
785 pub fn get_flags(&self) -> Result<u32> {
786 let mut flags = mem::MaybeUninit::uninit();
787 unsafe {
788 mdb_result(ffi::mdb_env_get_flags(
789 self.env_mut_ptr(),
790 flags.as_mut_ptr(),
791 ))?
792 };
793 let flags = unsafe { flags.assume_init() };
794 Ok(flags)
795 }
796
797 /// Returns some basic informations about this environment.
798 pub fn info(&self) -> EnvInfo {
799 let mut raw_info = mem::MaybeUninit::uninit();
800 unsafe { ffi::mdb_env_info(self.0.env, raw_info.as_mut_ptr()) };
801 let raw_info = unsafe { raw_info.assume_init() };
802
803 EnvInfo {
804 map_addr: raw_info.me_mapaddr,
805 map_size: raw_info.me_mapsize,
806 last_page_number: raw_info.me_last_pgno,
807 last_txn_id: raw_info.me_last_txnid,
808 maximum_number_of_readers: raw_info.me_maxreaders,
809 number_of_readers: raw_info.me_numreaders,
810 }
811 }
812
813 /// Returns the size used by all the databases in the environment without the free pages.
814 ///
815 /// It is crucial to configure [`EnvOpenOptions::max_dbs`] with a sufficiently large value
816 /// before invoking this function. All databases within the environment will be opened
817 /// and remain so.
818 pub fn non_free_pages_size(&self) -> Result<u64> {
819 let compute_size = |stat: ffi::MDB_stat| {
820 (stat.ms_leaf_pages + stat.ms_branch_pages + stat.ms_overflow_pages) as u64
821 * stat.ms_psize as u64
822 };
823
824 let mut size = 0;
825
826 let mut stat = mem::MaybeUninit::uninit();
827 unsafe { mdb_result(ffi::mdb_env_stat(self.env_mut_ptr(), stat.as_mut_ptr()))? };
828 let stat = unsafe { stat.assume_init() };
829 size += compute_size(stat);
830
831 let rtxn = self.read_txn()?;
832 // Open the main database
833 let dbi = self.raw_open_dbi::<DefaultComparator>(rtxn.txn, None, 0)?;
834
835 // We're going to iterate on the unnamed database
836 let mut cursor = RoCursor::new(&rtxn, dbi)?;
837
838 while let Some((key, _value)) = cursor.move_on_next(MoveOperation::NoDup)? {
839 if key.contains(&0) {
840 continue;
841 }
842
843 let key = String::from_utf8(key.to_vec()).unwrap();
844 // Calling `ffi::db_stat` on a database instance does not involve key comparison
845 // in LMDB, so it's safe to specify a noop key compare function for it.
846 if let Ok(dbi) = self.raw_open_dbi::<DefaultComparator>(rtxn.txn, Some(&key), 0) {
847 let mut stat = mem::MaybeUninit::uninit();
848 unsafe { mdb_result(ffi::mdb_stat(rtxn.txn, dbi, stat.as_mut_ptr()))? };
849 let stat = unsafe { stat.assume_init() };
850 size += compute_size(stat);
851 }
852 }
853
854 Ok(size)
855 }
856
857 /// Options and flags which can be used to configure how a [`Database`] is opened.
858 pub fn database_options(&self) -> DatabaseOpenOptions<'_, '_, Unspecified, Unspecified> {
859 DatabaseOpenOptions::new(self)
860 }
861
862 /// Opens a typed database that already exists in this environment.
863 ///
864 /// If the database was previously opened in this program run, types will be checked.
865 ///
866 /// ## Important Information
867 ///
868 /// LMDB has an important restriction on the unnamed database when named ones are opened.
869 /// The names of the named databases are stored as keys in the unnamed one and are immutable,
870 /// and these keys can only be read and not written.
871 ///
872 /// ## LMDB read-only access of existing database
873 ///
874 /// In the case of accessing a database in a read-only manner from another process
875 /// where you wrote, you might need to manually call [`RoTxn::commit`] to get metadata
876 /// and the database handles opened and shared with the global [`Env`] handle.
877 ///
878 /// If not done, you might raise `Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" })`
879 /// known as `EINVAL`.
880 pub fn open_database<KC, DC>(
881 &self,
882 rtxn: &RoTxn,
883 name: Option<&str>,
884 ) -> Result<Option<Database<KC, DC>>>
885 where
886 KC: 'static,
887 DC: 'static,
888 {
889 let mut options = self.database_options().types::<KC, DC>();
890 if let Some(name) = name {
891 options.name(name);
892 }
893 options.open(rtxn)
894 }
895
896 /// Creates a typed database that can already exist in this environment.
897 ///
898 /// If the database was previously opened during this program run, types will be checked.
899 ///
900 /// ## Important Information
901 ///
902 /// LMDB has an important restriction on the unnamed database when named ones are opened.
903 /// The names of the named databases are stored as keys in the unnamed one and are immutable,
904 /// and these keys can only be read and not written.
905 pub fn create_database<KC, DC>(
906 &self,
907 wtxn: &mut RwTxn,
908 name: Option<&str>,
909 ) -> Result<Database<KC, DC>>
910 where
911 KC: 'static,
912 DC: 'static,
913 {
914 let mut options = self.database_options().types::<KC, DC>();
915 if let Some(name) = name {
916 options.name(name);
917 }
918 options.create(wtxn)
919 }
920
921 pub(crate) fn raw_init_database<C: Comparator + 'static>(
922 &self,
923 raw_txn: *mut ffi::MDB_txn,
924 name: Option<&str>,
925 flags: AllDatabaseFlags,
926 ) -> Result<u32> {
927 match self.raw_open_dbi::<C>(raw_txn, name, flags.bits()) {
928 Ok(dbi) => Ok(dbi),
929 Err(e) => Err(e.into()),
930 }
931 }
932
933 fn raw_open_dbi<C: Comparator + 'static>(
934 &self,
935 raw_txn: *mut ffi::MDB_txn,
936 name: Option<&str>,
937 flags: u32,
938 ) -> std::result::Result<u32, crate::mdb::lmdb_error::Error> {
939 let mut dbi = 0;
940 let name = name.map(|n| CString::new(n).unwrap());
941 let name_ptr = match name {
942 Some(ref name) => name.as_bytes_with_nul().as_ptr() as *const _,
943 None => ptr::null(),
944 };
945
946 // safety: The name cstring is cloned by LMDB, we can drop it after.
947 // If a read-only is used with the MDB_CREATE flag, LMDB will throw an error.
948 unsafe {
949 mdb_result(ffi::mdb_dbi_open(raw_txn, name_ptr, flags, &mut dbi))?;
950 if TypeId::of::<C>() != TypeId::of::<DefaultComparator>() {
951 mdb_result(ffi::mdb_set_compare(
952 raw_txn,
953 dbi,
954 Some(custom_key_cmp_wrapper::<C>),
955 ))?;
956 }
957 };
958
959 Ok(dbi)
960 }
961
962 /// Create a transaction with read and write access for use with the environment.
963 ///
964 /// ## LMDB Limitations
965 ///
966 /// Only one [`RwTxn`] may exist simultaneously in the current environment.
967 /// If another write transaction is initiated, while another write transaction exists
968 /// the thread initiating the new one will wait on a mutex upon completion of the previous
969 /// transaction.
970 pub fn write_txn(&self) -> Result<RwTxn<'_>> {
971 RwTxn::new(self)
972 }
973
974 /// Create a nested transaction with read and write access for use with the environment.
975 ///
976 /// The new transaction will be a nested transaction, with the transaction indicated by parent
977 /// as its parent. Transactions may be nested to any level.
978 ///
979 /// A parent transaction and its cursors may not issue any other operations than _commit_ and
980 /// _abort_ while it has active child transactions.
981 pub fn nested_write_txn<'p>(&'p self, parent: &'p mut RwTxn) -> Result<RwTxn<'p>> {
982 RwTxn::nested(self, parent)
983 }
984
985 /// Create a transaction with read-only access for use with the environment.
986 ///
987 /// You can make this transaction `Send`able between threads by
988 /// using the `read-txn-no-tls` crate feature.
989 /// See [`Self::static_read_txn`] if you want the txn to own the environment.
990 ///
991 /// ## LMDB Limitations
992 ///
993 /// It's possible to have multiple read transactions in the same environment
994 /// while there is a write transaction ongoing.
995 ///
996 /// But read transactions prevent reuse of pages freed by newer write transactions,
997 /// thus the database can grow quickly. Write transactions prevent other write transactions,
998 /// since writes are serialized.
999 ///
1000 /// So avoid long-lived read transactions.
1001 ///
1002 /// ## Errors
1003 ///
1004 /// * [`crate::MdbError::Panic`]: A fatal error occurred earlier, and the environment must be shut down
1005 /// * [`crate::MdbError::MapResized`]: Another process wrote data beyond this [`Env`] mapsize and this env
1006 /// map must be resized
1007 /// * [`crate::MdbError::ReadersFull`]: a read-only transaction was requested, and the reader lock table is
1008 /// full
1009 pub fn read_txn(&self) -> Result<RoTxn<'_>> {
1010 RoTxn::new(self)
1011 }
1012
1013 /// Create a transaction with read-only access for use with the environment.
1014 /// Contrary to [`Self::read_txn`], this version **owns** the environment, which
1015 /// means you won't be able to close the environment while this transaction is alive.
1016 ///
1017 /// You can make this transaction `Send`able between threads by
1018 /// using the `read-txn-no-tls` crate feature.
1019 ///
1020 /// ## LMDB Limitations
1021 ///
1022 /// It's possible to have multiple read transactions in the same environment
1023 /// while there is a write transaction ongoing.
1024 ///
1025 /// But read transactions prevent reuse of pages freed by newer write transactions,
1026 /// thus the database can grow quickly. Write transactions prevent other write transactions,
1027 /// since writes are serialized.
1028 ///
1029 /// So avoid long-lived read transactions.
1030 ///
1031 /// ## Errors
1032 ///
1033 /// * [`crate::MdbError::Panic`]: A fatal error occurred earlier, and the environment must be shut down
1034 /// * [`crate::MdbError::MapResized`]: Another process wrote data beyond this [`Env`] mapsize and this env
1035 /// map must be resized
1036 /// * [`crate::MdbError::ReadersFull`]: a read-only transaction was requested, and the reader lock table is
1037 /// full
1038 pub fn static_read_txn(self) -> Result<RoTxn<'static>> {
1039 RoTxn::static_read_txn(self)
1040 }
1041
1042 /// Copy an LMDB environment to the specified path, with options.
1043 ///
1044 /// This function may be used to make a backup of an existing environment.
1045 /// No lockfile is created, since it gets recreated at need.
1046 pub fn copy_to_file<P: AsRef<Path>>(&self, path: P, option: CompactionOption) -> Result<File> {
1047 let file = File::options().create_new(true).write(true).open(&path)?;
1048 let fd = get_file_fd(&file);
1049
1050 unsafe { self.copy_to_fd(fd, option)? };
1051
1052 // We reopen the file to make sure the cursor is at the start,
1053 // even a seek to start doesn't work properly.
1054 let file = File::open(path)?;
1055
1056 Ok(file)
1057 }
1058
1059 /// Copy an LMDB environment to the specified file descriptor, with compaction option.
1060 ///
1061 /// This function may be used to make a backup of an existing environment.
1062 /// No lockfile is created, since it gets recreated at need.
1063 ///
1064 /// # Safety
1065 ///
1066 /// The [`ffi::mdb_filehandle_t`] must have already been opened for Write access.
1067 pub unsafe fn copy_to_fd(
1068 &self,
1069 fd: ffi::mdb_filehandle_t,
1070 option: CompactionOption,
1071 ) -> Result<()> {
1072 let flags = if let CompactionOption::Enabled = option {
1073 ffi::MDB_CP_COMPACT
1074 } else {
1075 0
1076 };
1077 mdb_result(ffi::mdb_env_copyfd2(self.0.env, fd, flags))?;
1078 Ok(())
1079 }
1080
1081 /// Flush the data buffers to disk.
1082 pub fn force_sync(&self) -> Result<()> {
1083 unsafe { mdb_result(ffi::mdb_env_sync(self.0.env, 1))? }
1084 Ok(())
1085 }
1086
1087 /// Returns the canonicalized path where this env lives.
1088 pub fn path(&self) -> &Path {
1089 &self.0.path
1090 }
1091
1092 /// Returns an `EnvClosingEvent` that can be used to wait for the closing event,
1093 /// multiple threads can wait on this event.
1094 ///
1095 /// Make sure that you drop all the copies of `Env`s you have, env closing are triggered
1096 /// when all references are dropped, the last one will eventually close the environment.
1097 pub fn prepare_for_closing(self) -> EnvClosingEvent {
1098 let mut lock = OPENED_ENV.write().unwrap();
1099 match lock.get_mut(self.path()) {
1100 None => panic!("cannot find the env that we are trying to close"),
1101 Some(EnvEntry {
1102 env, signal_event, ..
1103 }) => {
1104 // We remove the env from the global list and replace it with a None.
1105 let _env = env.take();
1106 let signal_event = signal_event.clone();
1107
1108 // we must make sure we release the lock before we drop the env
1109 // as the drop of the EnvInner also tries to lock the OPENED_ENV
1110 // global and we don't want to trigger a dead-lock.
1111 drop(lock);
1112
1113 EnvClosingEvent(signal_event)
1114 }
1115 }
1116 }
1117
1118 /// Check for stale entries in the reader lock table and clear them.
1119 ///
1120 /// Returns the number of stale readers cleared.
1121 pub fn clear_stale_readers(&self) -> Result<usize> {
1122 let mut dead: i32 = 0;
1123 unsafe { mdb_result(ffi::mdb_reader_check(self.0.env, &mut dead))? }
1124 // safety: The reader_check function asks for an i32, initialize it to zero
1125 // and never decrements it. It is safe to use either an u32 or u64 (usize).
1126 Ok(dead as usize)
1127 }
1128
1129 /// Resize the memory map to a new size.
1130 ///
1131 /// # Safety
1132 ///
1133 /// According to the [LMDB documentation](http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5),
1134 /// it is okay to call `mdb_env_set_mapsize` for an open environment as long as no transactions are active,
1135 /// but the library does not check for this condition, so the caller must ensure it explicitly.
1136 pub unsafe fn resize(&self, new_size: usize) -> Result<()> {
1137 if new_size % page_size::get() != 0 {
1138 let msg = format!(
1139 "map size ({}) must be a multiple of the system page size ({})",
1140 new_size,
1141 page_size::get()
1142 );
1143 return Err(Error::Io(io::Error::new(io::ErrorKind::InvalidInput, msg)));
1144 }
1145 mdb_result(unsafe { ffi::mdb_env_set_mapsize(self.env_mut_ptr(), new_size) })
1146 .map_err(Into::into)
1147 }
1148
1149 /// Get the maximum size of keys and MDB_DUPSORT data we can write.
1150 ///
1151 /// Depends on the compile-time constant MDB_MAXKEYSIZE. Default 511
1152 pub fn max_key_size(&self) -> usize {
1153 let maxsize: i32 = unsafe { ffi::mdb_env_get_maxkeysize(self.env_mut_ptr()) };
1154 maxsize as usize
1155 }
1156}
1157
1158/// Contains information about the environment.
1159#[derive(Debug, Clone, Copy)]
1160pub struct EnvInfo {
1161 /// Address of the map, if fixed.
1162 pub map_addr: *mut c_void,
1163 /// Size of the data memory map.
1164 pub map_size: usize,
1165 /// ID of the last used page.
1166 pub last_page_number: usize,
1167 /// ID of the last committed transaction.
1168 pub last_txn_id: usize,
1169 /// Maximum number of reader slots in the environment.
1170 pub maximum_number_of_readers: u32,
1171 /// Number of reader slots used in the environment.
1172 pub number_of_readers: u32,
1173}
1174
1175/// A structure that can be used to wait for the closing event.
1176/// Multiple threads can wait on this event.
1177#[derive(Clone)]
1178pub struct EnvClosingEvent(Arc<SignalEvent>);
1179
1180impl EnvClosingEvent {
1181 /// Blocks this thread until the environment is effectively closed.
1182 ///
1183 /// # Safety
1184 ///
1185 /// Make sure that you don't have any copy of the environment in the thread
1186 /// that is waiting for a close event. If you do, you will have a deadlock.
1187 pub fn wait(&self) {
1188 self.0.wait()
1189 }
1190
1191 /// Blocks this thread until either the environment has been closed
1192 /// or until the timeout elapses. Returns `true` if the environment
1193 /// has been effectively closed.
1194 pub fn wait_timeout(&self, timeout: Duration) -> bool {
1195 self.0.wait_timeout(timeout)
1196 }
1197}
1198
1199impl fmt::Debug for EnvClosingEvent {
1200 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1201 f.debug_struct("EnvClosingEvent").finish()
1202 }
1203}
1204
1205#[cfg(test)]
1206mod tests {
1207 use std::io::ErrorKind;
1208 use std::time::Duration;
1209 use std::{fs, thread};
1210
1211 use crate::types::*;
1212 use crate::{env_closing_event, EnvOpenOptions, Error};
1213
1214 #[test]
1215 fn close_env() {
1216 let dir = tempfile::tempdir().unwrap();
1217 let env = unsafe {
1218 EnvOpenOptions::new()
1219 .map_size(10 * 1024 * 1024) // 10MB
1220 .max_dbs(30)
1221 .open(dir.path())
1222 .unwrap()
1223 };
1224
1225 // Force a thread to keep the env for 1 second.
1226 let env_cloned = env.clone();
1227 thread::spawn(move || {
1228 let _env = env_cloned;
1229 thread::sleep(Duration::from_secs(1));
1230 });
1231
1232 let mut wtxn = env.write_txn().unwrap();
1233 let db = env.create_database::<Str, Str>(&mut wtxn, None).unwrap();
1234 wtxn.commit().unwrap();
1235
1236 // Create an ordered list of keys...
1237 let mut wtxn = env.write_txn().unwrap();
1238 db.put(&mut wtxn, "hello", "hello").unwrap();
1239 db.put(&mut wtxn, "world", "world").unwrap();
1240
1241 let mut iter = db.iter(&wtxn).unwrap();
1242 assert_eq!(iter.next().transpose().unwrap(), Some(("hello", "hello")));
1243 assert_eq!(iter.next().transpose().unwrap(), Some(("world", "world")));
1244 assert_eq!(iter.next().transpose().unwrap(), None);
1245 drop(iter);
1246
1247 wtxn.commit().unwrap();
1248
1249 let signal_event = env.prepare_for_closing();
1250
1251 eprintln!("waiting for the env to be closed");
1252 signal_event.wait();
1253 eprintln!("env closed successfully");
1254
1255 // Make sure we don't have a reference to the env
1256 assert!(env_closing_event(dir.path()).is_none());
1257 }
1258
1259 #[test]
1260 fn reopen_env_with_different_options_is_err() {
1261 let dir = tempfile::tempdir().unwrap();
1262 let _env = unsafe {
1263 EnvOpenOptions::new()
1264 .map_size(10 * 1024 * 1024) // 10MB
1265 .open(dir.path())
1266 .unwrap()
1267 };
1268
1269 let result = unsafe {
1270 EnvOpenOptions::new()
1271 .map_size(12 * 1024 * 1024) // 12MB
1272 .open(dir.path())
1273 };
1274
1275 assert!(matches!(result, Err(Error::BadOpenOptions { .. })));
1276 }
1277
1278 #[test]
1279 fn open_env_with_named_path() {
1280 let dir = tempfile::tempdir().unwrap();
1281 fs::create_dir_all(dir.path().join("babar.mdb")).unwrap();
1282 let _env = unsafe {
1283 EnvOpenOptions::new()
1284 .map_size(10 * 1024 * 1024) // 10MB
1285 .open(dir.path().join("babar.mdb"))
1286 .unwrap()
1287 };
1288
1289 let _env = unsafe {
1290 EnvOpenOptions::new()
1291 .map_size(10 * 1024 * 1024) // 10MB
1292 .open(dir.path().join("babar.mdb"))
1293 .unwrap()
1294 };
1295 }
1296
1297 #[test]
1298 #[cfg(not(windows))]
1299 fn open_database_with_writemap_flag() {
1300 let dir = tempfile::tempdir().unwrap();
1301 let mut envbuilder = EnvOpenOptions::new();
1302 envbuilder.map_size(10 * 1024 * 1024); // 10MB
1303 envbuilder.max_dbs(10);
1304 unsafe { envbuilder.flags(crate::EnvFlags::WRITE_MAP) };
1305 let env = unsafe { envbuilder.open(dir.path()).unwrap() };
1306
1307 let mut wtxn = env.write_txn().unwrap();
1308 let _db = env
1309 .create_database::<Str, Str>(&mut wtxn, Some("my-super-db"))
1310 .unwrap();
1311 wtxn.commit().unwrap();
1312 }
1313
1314 #[test]
1315 fn open_database_with_nosubdir() {
1316 let dir = tempfile::tempdir().unwrap();
1317 let mut envbuilder = EnvOpenOptions::new();
1318 unsafe { envbuilder.flags(crate::EnvFlags::NO_SUB_DIR) };
1319 let _env = unsafe { envbuilder.open(dir.path().join("data.mdb")).unwrap() };
1320 }
1321
1322 #[test]
1323 fn create_database_without_commit() {
1324 let dir = tempfile::tempdir().unwrap();
1325 let env = unsafe {
1326 EnvOpenOptions::new()
1327 .map_size(10 * 1024 * 1024) // 10MB
1328 .max_dbs(10)
1329 .open(dir.path())
1330 .unwrap()
1331 };
1332
1333 let mut wtxn = env.write_txn().unwrap();
1334 let _db = env
1335 .create_database::<Str, Str>(&mut wtxn, Some("my-super-db"))
1336 .unwrap();
1337 wtxn.abort();
1338
1339 let rtxn = env.read_txn().unwrap();
1340 let option = env
1341 .open_database::<Str, Str>(&rtxn, Some("my-super-db"))
1342 .unwrap();
1343 assert!(option.is_none());
1344 }
1345
1346 #[test]
1347 fn open_already_existing_database() {
1348 let dir = tempfile::tempdir().unwrap();
1349 let env = unsafe {
1350 EnvOpenOptions::new()
1351 .map_size(10 * 1024 * 1024) // 10MB
1352 .max_dbs(10)
1353 .open(dir.path())
1354 .unwrap()
1355 };
1356
1357 // we first create a database
1358 let mut wtxn = env.write_txn().unwrap();
1359 let _db = env
1360 .create_database::<Str, Str>(&mut wtxn, Some("my-super-db"))
1361 .unwrap();
1362 wtxn.commit().unwrap();
1363
1364 // Close the environement and reopen it, databases must not be loaded in memory.
1365 env.prepare_for_closing().wait();
1366 let env = unsafe {
1367 EnvOpenOptions::new()
1368 .map_size(10 * 1024 * 1024) // 10MB
1369 .max_dbs(10)
1370 .open(dir.path())
1371 .unwrap()
1372 };
1373
1374 let rtxn = env.read_txn().unwrap();
1375 let option = env
1376 .open_database::<Str, Str>(&rtxn, Some("my-super-db"))
1377 .unwrap();
1378 assert!(option.is_some());
1379 }
1380
1381 #[test]
1382 fn resize_database() {
1383 let dir = tempfile::tempdir().unwrap();
1384 let page_size = page_size::get();
1385 let env = unsafe {
1386 EnvOpenOptions::new()
1387 .map_size(9 * page_size)
1388 .max_dbs(1)
1389 .open(dir.path())
1390 .unwrap()
1391 };
1392
1393 let mut wtxn = env.write_txn().unwrap();
1394 let db = env
1395 .create_database::<Str, Str>(&mut wtxn, Some("my-super-db"))
1396 .unwrap();
1397 wtxn.commit().unwrap();
1398
1399 let mut wtxn = env.write_txn().unwrap();
1400 for i in 0..64 {
1401 db.put(&mut wtxn, &i.to_string(), "world").unwrap();
1402 }
1403 wtxn.commit().unwrap();
1404
1405 let mut wtxn = env.write_txn().unwrap();
1406 for i in 64..128 {
1407 db.put(&mut wtxn, &i.to_string(), "world").unwrap();
1408 }
1409 wtxn.commit()
1410 .expect_err("cannot commit a transaction that would reach the map size limit");
1411
1412 unsafe {
1413 env.resize(10 * page_size).unwrap();
1414 }
1415 let mut wtxn = env.write_txn().unwrap();
1416 for i in 64..128 {
1417 db.put(&mut wtxn, &i.to_string(), "world").unwrap();
1418 }
1419 wtxn.commit()
1420 .expect("transaction should commit after resizing the map size");
1421
1422 assert_eq!(10 * page_size, env.info().map_size);
1423 }
1424
1425 /// Non-regression test for
1426 /// <https://github.com/meilisearch/heed/issues/183>
1427 ///
1428 /// We should be able to open database Read-Only Env with
1429 /// no prior Read-Write Env opening. And query data.
1430 #[test]
1431 fn open_read_only_without_no_env_opened_before() {
1432 let expected_data0 = "Data Expected db0";
1433 let dir = tempfile::tempdir().unwrap();
1434
1435 {
1436 // We really need this env to be dropped before the read-only access.
1437 let env = unsafe {
1438 EnvOpenOptions::new()
1439 .map_size(16 * 1024 * 1024 * 1024) // 10MB
1440 .max_dbs(32)
1441 .open(dir.path())
1442 .unwrap()
1443 };
1444 let mut wtxn = env.write_txn().unwrap();
1445 let database0 = env
1446 .create_database::<Str, Str>(&mut wtxn, Some("shared0"))
1447 .unwrap();
1448
1449 wtxn.commit().unwrap();
1450 let mut wtxn = env.write_txn().unwrap();
1451 database0.put(&mut wtxn, "shared0", expected_data0).unwrap();
1452 wtxn.commit().unwrap();
1453 // We also really need that no other env reside in memory in other thread doing tests.
1454 env.prepare_for_closing().wait();
1455 }
1456
1457 {
1458 // Open now we do a read-only opening
1459 let env = unsafe {
1460 EnvOpenOptions::new()
1461 .map_size(16 * 1024 * 1024 * 1024) // 10MB
1462 .max_dbs(32)
1463 .open(dir.path())
1464 .unwrap()
1465 };
1466 let database0 = {
1467 let rtxn = env.read_txn().unwrap();
1468 let database0 = env
1469 .open_database::<Str, Str>(&rtxn, Some("shared0"))
1470 .unwrap()
1471 .unwrap();
1472 // This commit is mandatory if not committed you might get
1473 // Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" })
1474 rtxn.commit().unwrap();
1475 database0
1476 };
1477
1478 {
1479 // If we didn't committed the opening it might fail with EINVAL.
1480 let rtxn = env.read_txn().unwrap();
1481 let value = database0.get(&rtxn, "shared0").unwrap().unwrap();
1482 assert_eq!(value, expected_data0);
1483 }
1484
1485 env.prepare_for_closing().wait();
1486 }
1487
1488 // To avoid reintroducing the bug let's try to open again but without the commit
1489 {
1490 // Open now we do a read-only opening
1491 let env = unsafe {
1492 EnvOpenOptions::new()
1493 .map_size(16 * 1024 * 1024 * 1024) // 10MB
1494 .max_dbs(32)
1495 .open(dir.path())
1496 .unwrap()
1497 };
1498 let database0 = {
1499 let rtxn = env.read_txn().unwrap();
1500 let database0 = env
1501 .open_database::<Str, Str>(&rtxn, Some("shared0"))
1502 .unwrap()
1503 .unwrap();
1504 // No commit it's important, dropping explicitly
1505 drop(rtxn);
1506 database0
1507 };
1508
1509 {
1510 // We didn't committed the opening we will get EINVAL.
1511 let rtxn = env.read_txn().unwrap();
1512 // The dbg!() is intentional in case of a change in rust-std or in lmdb related
1513 // to the windows error.
1514 let err = dbg!(database0.get(&rtxn, "shared0"));
1515
1516 // The error kind is still ErrorKind Uncategorized on windows.
1517 // Behind it's a ERROR_BAD_COMMAND code 22 like EINVAL.
1518 if cfg!(windows) {
1519 assert!(err.is_err());
1520 } else {
1521 assert!(
1522 matches!(err, Err(Error::Io(ref e)) if e.kind() == ErrorKind::InvalidInput)
1523 );
1524 }
1525 }
1526
1527 env.prepare_for_closing().wait();
1528 }
1529 }
1530
1531 #[test]
1532 fn max_key_size() {
1533 let dir = tempfile::tempdir().unwrap();
1534 let env = unsafe {
1535 EnvOpenOptions::new()
1536 .open(dir.path().join(dir.path()))
1537 .unwrap()
1538 };
1539 let maxkeysize = env.max_key_size();
1540
1541 eprintln!("maxkeysize: {}", maxkeysize);
1542
1543 if cfg!(feature = "longer-keys") {
1544 // Should be larger than the default of 511
1545 assert!(maxkeysize > 511);
1546 } else {
1547 // Should be the default of 511
1548 assert_eq!(maxkeysize, 511);
1549 }
1550 }
1551}