Struct StorageOptions

pub struct StorageOptions {
    pub backend: StorageBackendConfig,
    pub cache_mib: Option<u64>,
    pub compression: StorageCompression,
    pub min_step_storage_bytes: Option<u64>,
    pub min_storage_bytes: Option<u64>,
}

Storage configuration for a pipeline.

JSON schema

{
 "description": "Storage configuration for a pipeline.",
 "type": "object",
 "properties": {
   "backend": {
     "default": {
       "name": "default"
     },
     "allOf": [
       {
         "$ref": "#/components/schemas/StorageBackendConfig"
       }
     ]
   },
   "cache_mib": {
     "description": "The maximum size of the in-memory storage cache, in MiB.\n\nIf set, the specified cache size is spread across all the foreground and\nbackground threads. If unset, each foreground or background thread cache\nis limited to 256 MiB.",
     "type": [
       "integer",
       "null"
     ],
     "minimum": 0.0
   },
   "compression": {
     "default": "default",
     "allOf": [
       {
         "$ref": "#/components/schemas/StorageCompression"
       }
     ]
   },
   "min_step_storage_bytes": {
     "description": "For a batch of data passed through the pipeline during a single step,\nthe minimum estimated number of bytes to write it to storage.\n\nThis is provided for debugging and fine-tuning and should ordinarily be\nleft unset.  A value of 0 will write even empty batches to storage, and\nnonzero values provide a threshold.  `usize::MAX`, the default,\neffectively disables storage for such batches.  If it is set to another\nvalue, it should ordinarily be greater than or equal to\n`min_storage_bytes`.",
     "type": [
       "integer",
       "null"
     ],
     "minimum": 0.0
   },
   "min_storage_bytes": {
     "description": "For a batch of data maintained as part of a persistent index during a\npipeline run, the minimum estimated number of bytes to write it to\nstorage.\n\nThis is provided for debugging and fine-tuning and should ordinarily be\nleft unset.\n\nA value of 0 will write even empty batches to storage, and nonzero\nvalues provide a threshold.  `usize::MAX` would effectively disable\nstorage for such batches.  The default is 1,048,576 (1 MiB).",
     "type": [
       "integer",
       "null"
     ],
     "minimum": 0.0
   }
 }
}

Fields

backend: StorageBackendConfig

cache_mib: Option<u64>

The maximum size of the in-memory storage cache, in MiB.

If set, the specified cache size is spread across all the foreground and background threads. If unset, each foreground or background thread cache is limited to 256 MiB.

compression: StorageCompression

min_step_storage_bytes: Option<u64>

For a batch of data passed through the pipeline during a single step, the minimum estimated number of bytes to write it to storage.

This is provided for debugging and fine-tuning and should ordinarily be left unset. A value of 0 will write even empty batches to storage, and nonzero values provide a threshold. usize::MAX, the default, effectively disables storage for such batches. If it is set to another value, it should ordinarily be greater than or equal to min_storage_bytes.

min_storage_bytes: Option<u64>

For a batch of data maintained as part of a persistent index during a pipeline run, the minimum estimated number of bytes to write it to storage.

This is provided for debugging and fine-tuning and should ordinarily be left unset.

A value of 0 will write even empty batches to storage, and nonzero values provide a threshold. usize::MAX would effectively disable storage for such batches. The default is 1,048,576 (1 MiB).

Implementations

impl StorageOptions

pub fn builder() -> StorageOptions

Trait Implementations

impl Clone for StorageOptions

fn clone(&self) -> StorageOptions

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for StorageOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for StorageOptions

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl From<&StorageOptions> for StorageOptions

fn from(value: &StorageOptions) -> Self

Converts to this type from the input type.

impl From<StorageOptions> for StorageOptions

fn from(value: StorageOptions) -> Self

Converts to this type from the input type.

impl Serialize for StorageOptions

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl TryFrom<StorageOptions> for StorageOptions

type Error = ConversionError

The type returned in the event of a conversion error.

fn try_from(value: StorageOptions) -> Result<Self, ConversionError>

Performs the conversion.