Struct StorageOptions 

pub struct StorageOptions {
    pub backend: StorageBackendConfig,
    pub bloom_false_positive_rate: Option<f64>,
    pub cache_mib: Option<u64>,
    pub compression: StorageCompression,
    pub min_step_storage_bytes: Option<u64>,
    pub min_storage_bytes: Option<u64>,
}

Storage configuration for a pipeline.

JSON schema

{
 "description": "Storage configuration for a pipeline.",
 "type": "object",
 "properties": {
   "backend": {
     "default": {
       "name": "default"
     },
     "allOf": [
       {
         "$ref": "#/components/schemas/StorageBackendConfig"
       }
     ]
   },
   "bloom_false_positive_rate": {
     "description": "False-positive rate for Bloom filters on batches on storage, as a\nfraction f, where 0 < f < 1.\n\nThe false-positive rate trades off between the amount of memory used by\nBloom filters and how frequently storage needs to be searched for keys\nthat are not actually present.  Typical false-positive rates and their\ncorresponding memory costs are:\n\n- 0.1: 4.8 bits per key\n- 0.01: 9.6 bits per key\n- 0.001: 14.4 bits per key\n- 0.0001: 19.2 bits per key (default)\n\nValues outside the valid range, such as 0.0, disable Bloom filters.",
     "type": [
       "number",
       "null"
     ],
     "format": "double"
   },
   "cache_mib": {
     "description": "The maximum size of the in-memory storage cache, in MiB.\n\nIf set, the specified cache size is spread across all the foreground and\nbackground threads. If unset, each foreground or background thread cache\nis limited to 256 MiB.",
     "type": [
       "integer",
       "null"
     ],
     "minimum": 0.0
   },
   "compression": {
     "default": "default",
     "allOf": [
       {
         "$ref": "#/components/schemas/StorageCompression"
       }
     ]
   },
   "min_step_storage_bytes": {
     "description": "For a batch of data passed through the pipeline during a single step,\nthe minimum estimated number of bytes to write it to storage.\n\nThis is provided for debugging and fine-tuning and should ordinarily be\nleft unset.  A value of 0 will write even empty batches to storage, and\nnonzero values provide a threshold.  `usize::MAX`, the default,\neffectively disables storage for such batches.  If it is set to another\nvalue, it should ordinarily be greater than or equal to\n`min_storage_bytes`.",
     "type": [
       "integer",
       "null"
     ],
     "minimum": 0.0
   },
   "min_storage_bytes": {
     "description": "For a batch of data maintained as part of a persistent index during a\npipeline run, the minimum estimated number of bytes to write it to\nstorage.\n\nThis is provided for debugging and fine-tuning and should ordinarily be\nleft unset.\n\nA value of 0 will write even empty batches to storage, and nonzero\nvalues provide a threshold.  `usize::MAX` would effectively disable\nstorage for such batches.  The default is 10,048,576 (10 MiB).",
     "type": [
       "integer",
       "null"
     ],
     "minimum": 0.0
   }
 }
}

Fields

backend: StorageBackendConfig

bloom_false_positive_rate: Option<f64>

False-positive rate for Bloom filters on batches on storage, as a fraction f, where 0 < f < 1.

The false-positive rate trades off between the amount of memory used by Bloom filters and how frequently storage needs to be searched for keys that are not actually present. Typical false-positive rates and their corresponding memory costs are:

• 0.1: 4.8 bits per key
• 0.01: 9.6 bits per key
• 0.001: 14.4 bits per key
• 0.0001: 19.2 bits per key (default)

Values outside the valid range, such as 0.0, disable Bloom filters.

cache_mib: Option<u64>

The maximum size of the in-memory storage cache, in MiB.

If set, the specified cache size is spread across all the foreground and background threads. If unset, each foreground or background thread cache is limited to 256 MiB.

compression: StorageCompression

min_step_storage_bytes: Option<u64>

For a batch of data passed through the pipeline during a single step, the minimum estimated number of bytes to write it to storage.

This is provided for debugging and fine-tuning and should ordinarily be left unset. A value of 0 will write even empty batches to storage, and nonzero values provide a threshold. usize::MAX, the default, effectively disables storage for such batches. If it is set to another value, it should ordinarily be greater than or equal to min_storage_bytes.

min_storage_bytes: Option<u64>

For a batch of data maintained as part of a persistent index during a pipeline run, the minimum estimated number of bytes to write it to storage.

This is provided for debugging and fine-tuning and should ordinarily be left unset.

A value of 0 will write even empty batches to storage, and nonzero values provide a threshold. usize::MAX would effectively disable storage for such batches. The default is 10,048,576 (10 MiB).

Implementations

impl StorageOptions

pub fn builder() -> StorageOptions

Trait Implementations

impl Clone for StorageOptions

fn clone(&self) -> StorageOptions

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for StorageOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for StorageOptions

fn default() -> Self

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for StorageOptions

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<&StorageOptions> for StorageOptions

fn from(value: &StorageOptions) -> Self

Converts to this type from the input type.

impl From<StorageOptions> for StorageOptions

fn from(value: StorageOptions) -> Self

Converts to this type from the input type.

impl Serialize for StorageOptions

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl TryFrom<StorageOptions> for StorageOptions

type Error = ConversionError

The type returned in the event of a conversion error.

fn try_from(value: StorageOptions) -> Result<Self, ConversionError>

Performs the conversion.