Struct CacheManagerConfig 

pub struct CacheManagerConfig {
    pub file_statistics_cache: Option<Arc<dyn FileStatisticsCache>>,
    pub file_statistics_cache_limit: usize,
    pub list_files_cache: Option<Arc<dyn ListFilesCache>>,
    pub list_files_cache_limit: usize,
    pub list_files_cache_ttl: Option<Duration>,
    pub file_metadata_cache: Option<Arc<dyn FileMetadataCache>>,
    pub metadata_cache_limit: usize,
}

Fields

file_statistics_cache: Option<Arc<dyn FileStatisticsCache>>

Enable caching of file statistics when listing files. Enabling the cache avoids repeatedly reading file statistics in a DataFusion session. Default is enabled. Currently only Parquet files are supported.

file_statistics_cache_limit: usize

Limit of the file statistics cache, in bytes. Default: 20MiB.

list_files_cache: Option<Arc<dyn ListFilesCache>>

Enable caching of file metadata when listing files. Enabling the cache avoids repeat list and object metadata fetch operations, which may be expensive in certain situations (e.g. remote object storage), for objects under paths that are cached. Note that if this option is enabled, DataFusion will not see any updates to the underlying storage for at least list_files_cache_ttl duration. Default is enabled.

list_files_cache_limit: usize

Limit of the list_files_cache, in bytes. Default: 1MiB.

list_files_cache_ttl: Option<Duration>

The duration the list files cache will consider an entry valid after insertion. Note that changes to the underlying storage system, such as adding or removing data, will not be visible until an entry expires. Default: None (infinite).

file_metadata_cache: Option<Arc<dyn FileMetadataCache>>

Cache of file-embedded metadata, used to avoid reading it multiple times when processing a data file (e.g., Parquet footer and page metadata). If not provided, the CacheManager will create a DefaultFilesMetadataCache.

metadata_cache_limit: usize

Limit of the file-embedded metadata cache, in bytes.

Implementations

impl CacheManagerConfig

pub fn with_file_statistics_cache( self, cache: Option<Arc<dyn FileStatisticsCache>>, ) -> Self

Set the cache for file statistics.

pub fn with_file_statistics_cache_limit(self, limit: usize) -> Self

Specifies the memory limit for the file statistics cache, in bytes.

pub fn with_list_files_cache( self, cache: Option<Arc<dyn ListFilesCache>>, ) -> Self

Set the cache for listing files.

Default is None (disabled).

pub fn with_list_files_cache_limit(self, limit: usize) -> Self

Sets the limit of the list files cache, in bytes.

Default: 1MiB (1,048,576 bytes).

pub fn with_list_files_cache_ttl(self, ttl: Option<Duration>) -> Self

Sets the TTL (time-to-live) for entries in the list files cache.

Default: None (infinite).

pub fn with_file_metadata_cache( self, cache: Option<Arc<dyn FileMetadataCache>>, ) -> Self

Sets the cache for file-embedded metadata.

Default is a DefaultFilesMetadataCache.

pub fn with_metadata_cache_limit(self, limit: usize) -> Self

Sets the limit of the file-embedded metadata cache, in bytes.

Trait Implementations

impl Clone for CacheManagerConfig

fn clone(&self) -> CacheManagerConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Default for CacheManagerConfig

fn default() -> Self

Returns the “default value” for a type.