algeon_datafusion/native_exec/config/
source_reads.rs

1//! Remote Parquet read settings, fixed when the execution config is built.
2
3use std::{
4    num::NonZeroU64,
5    path::{Path, PathBuf},
6};
7
8/// When a local byte-cache miss for an `s3://` Parquet object is filled.
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
10#[non_exhaustive]
11pub enum LakehouseByteCacheFill {
12    /// A miss reads the object remotely and leaves the cache unchanged.
13    #[default]
14    Disabled,
15    /// A miss reads the object remotely and fills the cache in the background.
16    ReadThrough,
17    /// A miss fills the cache first; the read then opens the local file.
18    FillBeforeRead,
19}
20
21/// Local byte cache for `s3://` Parquet objects.
22///
23/// Only an object with an ETag or version is served from or filled into the
24/// cache; its entry is keyed by that identity, the bucket and endpoint, and
25/// the object location.
26#[derive(Debug, Clone, PartialEq, Eq)]
27pub struct LakehouseByteCache {
28    root: PathBuf,
29    serves_hits: bool,
30    fill: LakehouseByteCacheFill,
31    max_retained_bytes: Option<NonZeroU64>,
32}
33
34impl LakehouseByteCache {
35    /// A cache under `root` that serves complete entries and never fills.
36    pub fn new(root: impl Into<PathBuf>) -> Self {
37        Self { root: root.into(), serves_hits: true, fill: LakehouseByteCacheFill::Disabled, max_retained_bytes: None }
38    }
39
40    /// Selects when a miss is filled.
41    pub fn with_fill(mut self, fill: LakehouseByteCacheFill) -> Self {
42        self.fill = fill;
43        self
44    }
45
46    /// Reads every object remotely even when a complete entry exists, for
47    /// benchmark isolation. Misses are still filled as [`Self::with_fill`]
48    /// selects.
49    pub fn ignoring_hits(mut self) -> Self {
50        self.serves_hits = false;
51        self
52    }
53
54    /// Best-effort byte target the cache root is trimmed toward.
55    pub fn with_max_retained_bytes(mut self, bytes: NonZeroU64) -> Self {
56        self.max_retained_bytes = Some(bytes);
57        self
58    }
59
60    pub(crate) fn root(&self) -> &Path {
61        &self.root
62    }
63
64    pub(crate) fn serves_hits(&self) -> bool {
65        self.serves_hits
66    }
67
68    pub(crate) fn fill(&self) -> LakehouseByteCacheFill {
69        self.fill
70    }
71
72    pub(crate) fn max_retained_bytes(&self) -> Option<NonZeroU64> {
73        self.max_retained_bytes
74    }
75}
algeon_datafusion/native_exec/config/source_reads.rs

algeon_datafusion/native_exec/config/
source_reads.rs
