rust_rocksdb/

lib.rs

// Copyright 2020 Tyler Neely
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

//! Rust wrapper for RocksDB.
//!
//! # Examples
//!
//! ```
//! use rust_rocksdb::{DB, Options};
//! // NB: db is automatically closed at end of lifetime
//! let tempdir = tempfile::Builder::new()
//!     .prefix("_path_for_rocksdb_storage")
//!     .tempdir()
//!     .expect("Failed to create temporary path for the _path_for_rocksdb_storage");
//! let path = tempdir.path();
//! {
//!    let db = DB::open_default(path).unwrap();
//!    db.put(b"my key", b"my value").unwrap();
//!    match db.get(b"my key") {
//!        Ok(Some(value)) => println!("retrieved value {}", String::from_utf8(value).unwrap()),
//!        Ok(None) => println!("value not found"),
//!        Err(e) => println!("operational problem encountered: {}", e),
//!    }
//!    db.delete(b"my key").unwrap();
//! }
//! let _ = DB::destroy(&Options::default(), path);
//! ```
//!
//! Opening a database and a single column family with custom options:
//!
//! ```
//! use rust_rocksdb::{DB, ColumnFamilyDescriptor, Options};
//!
//! let tempdir = tempfile::Builder::new()
//!     .prefix("_path_for_rocksdb_storage_with_cfs")
//!     .tempdir()
//!     .expect("Failed to create temporary path for the _path_for_rocksdb_storage_with_cfs.");
//! let path = tempdir.path();
//! let mut cf_opts = Options::default();
//! cf_opts.set_max_write_buffer_number(16);
//! let cf = ColumnFamilyDescriptor::new("cf1", cf_opts);
//!
//! let mut db_opts = Options::default();
//! db_opts.create_missing_column_families(true);
//! db_opts.create_if_missing(true);
//! {
//!     let db = DB::open_cf_descriptors(&db_opts, path, vec![cf]).unwrap();
//! }
//! let _ = DB::destroy(&db_opts, path);
//! ```
//!

// Only docs.rs passes `--cfg docsrs`, and only it builds on nightly, so this
// is inert everywhere else. It is what puts the "available on crate feature X
// only" badges on gated items. `auto_cfg` is on by default under this gate, so
// individual items do not need annotating.
#![cfg_attr(docsrs, feature(doc_cfg))]
#![warn(clippy::pedantic)]
#![allow(
    // Next `cast_*` lints don't give alternatives.
    clippy::cast_possible_wrap, clippy::cast_possible_truncation, clippy::cast_sign_loss,
    // Next lints produce too much noise/false positives.
    clippy::module_name_repetitions, clippy::similar_names, clippy::must_use_candidate,
    // '... may panic' lints.
    // Too much work to fix.
    clippy::missing_errors_doc,
    clippy::should_panic_without_expect,
    // False positive: WebSocket
    clippy::doc_markdown,
    clippy::missing_safety_doc,
    clippy::needless_pass_by_value,
    clippy::ptr_as_ptr,
    clippy::missing_panics_doc,
    clippy::from_over_into,
)]

#[macro_use]
mod ffi_util;

pub mod backup;
mod cache;
pub mod checkpoint;
mod column_family;
pub mod compaction;
pub mod compaction_filter;
pub mod compaction_filter_factory;
pub mod compaction_service;
mod comparator;
mod db;
mod db_iterator;
mod db_options;
mod db_pinnable_batch;
mod db_pinnable_slice;
mod env;
mod env_options;
pub mod event_listener;
pub mod file_checksum;
mod iter_range;
pub mod merge_operator;
pub mod metadata;
pub mod perf;
mod prop_name;
pub mod properties;
mod slice_transform;
mod snapshot;
pub mod sst_file_manager;
mod sst_file_writer;
pub mod sst_partitioner;
pub mod statistics;
pub mod table_properties;
pub mod trace;
mod transactions;
pub mod wal;
pub mod wal_filter;
mod write_batch;
mod write_batch_with_index;
mod write_buffer_manager;

pub use crate::{
    cache::{Cache, HyperClockCacheOptions, MemoryAllocator},
    column_family::{
        AsColumnFamilyRef, BoundColumnFamily, ColumnFamily, ColumnFamilyDescriptor,
        ColumnFamilyRef, ColumnFamilyTtl, DEFAULT_COLUMN_FAMILY_NAME,
    },
    compaction::{
        BlobFileAdditionInfo, BlobFileGarbageInfo, CompactionCancellationToken, CompactionFileInfo,
        CompactionJobStats, CompactionOptions,
    },
    compaction_filter::Decision as CompactionDecision,
    compaction_service::{
        CompactionService, CompactionServiceJobInfo, CompactionServiceJobStatus,
        CompactionServiceOptionsOverride, EnvPriority, OpenAndCompactCancellationToken,
        OpenAndCompactOptions, ScheduleResponse,
    },
    comparator::Comparator,
    db::{
        ColumnFamilyMetaData, CompactFilesResult, DB, DBAccess, DBCommon, DBWithThreadMode,
        ExportImportFilesMetaData, GetIntoBufferResult, LiveFile, MultiThreaded, OwnedPrefixProber,
        PrefixProber, Range, SingleThreaded, ThreadMode, TimestampedValue,
    },
    db_iterator::{
        DBIterator, DBIteratorWithThreadMode, DBRawIterator, DBRawIteratorWithThreadMode,
        DBWALIterator, Direction, IteratorMode,
    },
    db_options::{
        BlockBasedIndexType, BlockBasedOptions, BlockBasedPinningTier, BottommostLevelCompaction,
        ChecksumType, CompactOptions, CuckooTableOptions, DBCompactionPri, DBCompactionStyle,
        DBCompressionType, DBPath, DBRecoveryMode, DataBlockIndexType, FifoCompactOptions,
        FlushOptions, FlushWalOptions, ImportColumnFamilyOptions, IndexBlockSearchType, InfoLogger,
        IngestExternalFileOptions, KeyEncodingType, LogLevel, LruCacheOptions, MemtableFactory,
        Options, PlainTableFactoryOptions, PrepopulateBlobCache, RateLimiterMode, ReadOptions,
        ReadTier, SizeApproximationFlags, SizeApproximationOptions, UniversalCompactOptions,
        UniversalCompactionStopStyle, WaitForCompactOptions, WriteOptions,
    },
    db_pinnable_batch::{DBPinnableBatch, DBPinnableBatchIter},
    db_pinnable_slice::DBPinnableSlice,
    env::{Env, IoPriority},
    env_options::EnvOptions,
    event_listener::OwnedCompactionJobInfo,
    ffi_util::{CSlice, CStrLike},
    file_checksum::FileChecksumGenFactory,
    iter_range::{IterateBounds, PrefixRange},
    merge_operator::MergeOperands,
    metadata::{
        ColumnFamilyMetaDataOptions, FileType, LevelMetaData, LiveFileStorageInfoEntry,
        LiveFilesStorageInfo, LiveFilesStorageInfoOptions, SstFileMetaData, Temperature,
    },
    perf::{PerfContext, PerfMetric, PerfStatsLevel, with_thread_local},
    slice_transform::SliceTransform,
    snapshot::{Snapshot, SnapshotReadOptions, SnapshotWithThreadMode},
    sst_file_manager::SstFileManager,
    sst_file_writer::SstFileWriter,
    sst_partitioner::SstPartitionerFactory,
    table_properties::TableProperties,
    trace::{
        BlockCacheTraceOptions, BlockCacheTraceWriterOptions, ReplayOptions, Replayer, TraceFilter,
        TraceOptions, TraceReader,
    },
    transactions::{
        OccLockBuckets, OccValidationPolicy, OptimisticTransactionDB,
        OptimisticTransactionDBOptions, OptimisticTransactionOptions, Transaction, TransactionDB,
        TransactionDBOptions, TransactionOptions, TxnDBWritePolicy,
    },
    wal::{OwnedWalFile, WalFile, WalFileType, WalFiles, WalReadOptions},
    wal_filter::{WalFilter, WalRecordAction},
    write_batch::{
        WriteBatch, WriteBatchIterator, WriteBatchIteratorCf, WriteBatchWithTransaction,
    },
    write_batch_with_index::WriteBatchWithIndex,
    write_buffer_manager::WriteBufferManager,
};

use rust_librocksdb_sys as ffi;

/// The raw `librocksdb` bindings this crate is built against.
///
/// [`AsRawPtr`] hands out pointers to types from this crate, such as
/// `ffi::rocksdb_t`, so callers need a way to name them. Reaching them through
/// a separate `rust-librocksdb-sys` dependency would mean keeping that
/// version in lockstep with this crate's by hand, and every sys bump would
/// break it. Going through this re-export keeps the two tied together.
///
/// Everything here is generated by bindgen and is not covered by this crate's
/// semver guarantee. It changes whenever the vendored RocksDB does.
#[cfg(feature = "raw-ptr")]
pub use rust_librocksdb_sys as ffi_raw;

/// Returns `true` if this crate was built with the `coroutines` feature, in
/// which case librocksdb was compiled with `USE_COROUTINES` and linked
/// against folly.
///
/// When `true`, calling [`ReadOptions::set_async_io(true)`][async-io] on a
/// `MultiGet` activates the multi-level parallel-read path described in the
/// RocksDB [Asynchronous IO blog post]. When `false`, `MultiGet` with
/// `async_io=true` can only parallelize reads within a single LSM level.
///
/// Note: this reflects how this crate was configured, not what is in the
/// linked `librocksdb`. If you used `ROCKSDB_LIB_DIR` to link against an
/// externally-built `librocksdb.a`, the answer here may not match what that
/// library was actually compiled with.
///
/// [async-io]: ReadOptions::set_async_io
/// [Asynchronous IO blog post]: https://rocksdb.org/blog/2022/10/07/asynchronous-io-in-rocksdb.html
#[must_use]
pub fn built_with_coroutines() -> bool {
    cfg!(feature = "coroutines")
}

#[cfg(feature = "raw-ptr")]
mod raw_ptr;

#[cfg(feature = "raw-ptr")]
pub use crate::raw_ptr::AsRawPtr;

use std::error;
use std::fmt;

/// RocksDB error kind.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ErrorKind {
    NotFound,
    Corruption,
    NotSupported,
    InvalidArgument,
    IOError,
    MergeInProgress,
    Incomplete,
    ShutdownInProgress,
    TimedOut,
    Aborted,
    Busy,
    Expired,
    TryAgain,
    CompactionTooLarge,
    ColumnFamilyDropped,
    Unknown,
}

/// A simple wrapper round a string, used for errors reported from
/// ffi calls.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Error {
    message: String,
}

impl Error {
    fn new(message: String) -> Error {
        Error { message }
    }

    pub fn into_string(self) -> String {
        self.into()
    }

    /// Parse corresponding [`ErrorKind`] from error message.
    pub fn kind(&self) -> ErrorKind {
        match self.message.split(':').next().unwrap_or("") {
            "NotFound" => ErrorKind::NotFound,
            "Corruption" => ErrorKind::Corruption,
            "Not implemented" => ErrorKind::NotSupported,
            "Invalid argument" => ErrorKind::InvalidArgument,
            "IO error" => ErrorKind::IOError,
            "Merge in progress" => ErrorKind::MergeInProgress,
            "Result incomplete" => ErrorKind::Incomplete,
            "Shutdown in progress" => ErrorKind::ShutdownInProgress,
            "Operation timed out" => ErrorKind::TimedOut,
            "Operation aborted" => ErrorKind::Aborted,
            "Resource busy" => ErrorKind::Busy,
            "Operation expired" => ErrorKind::Expired,
            "Operation failed. Try again." => ErrorKind::TryAgain,
            "Compaction too large" => ErrorKind::CompactionTooLarge,
            "Column family dropped" => ErrorKind::ColumnFamilyDropped,
            _ => ErrorKind::Unknown,
        }
    }
}

impl AsRef<str> for Error {
    fn as_ref(&self) -> &str {
        &self.message
    }
}

impl From<Error> for String {
    fn from(e: Error) -> String {
        e.message
    }
}

impl error::Error for Error {
    fn description(&self) -> &str {
        &self.message
    }
}

impl fmt::Display for Error {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        self.message.fmt(formatter)
    }
}

#[cfg(test)]
mod test {
    use crate::{
        OptimisticTransactionDB, OptimisticTransactionOptions, Transaction, TransactionDB,
        TransactionDBOptions, TransactionOptions,
        cache::{Cache, CacheWrapper},
        write_buffer_manager::{WriteBufferManager, WriteBufferManagerWrapper},
    };

    use super::{
        BlockBasedOptions, BoundColumnFamily, ColumnFamily, ColumnFamilyDescriptor, DB, DBIterator,
        DBRawIterator, IngestExternalFileOptions, Options, PlainTableFactoryOptions, ReadOptions,
        Snapshot, SstFileWriter, WriteBatch, WriteOptions,
        column_family::UnboundColumnFamily,
        env::{Env, EnvWrapper},
    };

    #[test]
    fn is_send() {
        // test (at compile time) that certain types implement the auto-trait Send, either directly for
        // pointer-wrapping types or transitively for types with all Send fields

        fn is_send<T: Send>() {
            // dummy function just used for its parameterized type bound
        }

        is_send::<DB>();
        is_send::<DBIterator<'_>>();
        is_send::<DBRawIterator<'_>>();
        is_send::<Snapshot>();
        is_send::<Options>();
        is_send::<ReadOptions>();
        is_send::<WriteOptions>();
        is_send::<IngestExternalFileOptions>();
        is_send::<BlockBasedOptions>();
        is_send::<PlainTableFactoryOptions>();
        is_send::<ColumnFamilyDescriptor>();
        is_send::<ColumnFamily>();
        is_send::<BoundColumnFamily<'_>>();
        is_send::<UnboundColumnFamily>();
        is_send::<SstFileWriter>();
        is_send::<WriteBatch>();
        is_send::<Cache>();
        is_send::<CacheWrapper>();
        is_send::<Env>();
        is_send::<EnvWrapper>();
        is_send::<TransactionDB>();
        is_send::<OptimisticTransactionDB>();
        is_send::<Transaction<'_, TransactionDB>>();
        is_send::<TransactionDBOptions>();
        is_send::<OptimisticTransactionOptions>();
        is_send::<TransactionOptions>();
        is_send::<WriteBufferManager>();
        is_send::<WriteBufferManagerWrapper>();
    }

    #[test]
    fn is_sync() {
        // test (at compile time) that certain types implement the auto-trait Sync

        fn is_sync<T: Sync>() {
            // dummy function just used for its parameterized type bound
        }

        is_sync::<DB>();
        is_sync::<Snapshot>();
        is_sync::<Options>();
        is_sync::<ReadOptions>();
        is_sync::<WriteOptions>();
        is_sync::<IngestExternalFileOptions>();
        is_sync::<BlockBasedOptions>();
        is_sync::<PlainTableFactoryOptions>();
        is_sync::<UnboundColumnFamily>();
        is_sync::<ColumnFamilyDescriptor>();
        is_sync::<ColumnFamily>();
        is_sync::<SstFileWriter>();
        is_sync::<Cache>();
        is_sync::<CacheWrapper>();
        is_sync::<Env>();
        is_sync::<EnvWrapper>();
        is_sync::<TransactionDB>();
        is_sync::<OptimisticTransactionDB>();
        is_sync::<TransactionDBOptions>();
        is_sync::<OptimisticTransactionOptions>();
        is_sync::<TransactionOptions>();
        is_sync::<WriteBufferManager>();
        is_sync::<WriteBufferManagerWrapper>();
    }
}