floe_core/io/write/delta/
options.rs

1use crate::errors::RunError;
2use crate::io::write::metrics;
3use crate::{config, FloeResult};
4
5#[derive(Debug, Clone, PartialEq, Eq)]
6pub struct DeltaWriteRuntimeOptions {
7    pub partition_by: Option<Vec<String>>,
8    pub target_file_size_bytes: Option<usize>,
9    pub small_file_threshold_bytes: u64,
10}
11
12pub fn delta_write_runtime_options(
13    entity: &config::EntityConfig,
14) -> FloeResult<DeltaWriteRuntimeOptions> {
15    let target_file_size_bytes_u64 = entity
16        .sink
17        .accepted
18        .options
19        .as_ref()
20        .and_then(|options| options.max_size_per_file);
21    let target_file_size_bytes = match target_file_size_bytes_u64 {
22        Some(value) => Some(usize::try_from(value).map_err(|_| {
23            Box::new(RunError(format!(
24                "delta sink max_size_per_file is too large for this platform: {value}"
25            ))) as Box<dyn std::error::Error + Send + Sync>
26        })?),
27        None => None,
28    };
29    Ok(DeltaWriteRuntimeOptions {
30        partition_by: entity.sink.accepted.partition_by.clone(),
31        target_file_size_bytes,
32        small_file_threshold_bytes: metrics::default_small_file_threshold_bytes(
33            target_file_size_bytes_u64,
34        ),
35    })
36}
floe_core/io/write/delta/options.rs

floe_core/io/write/delta/
options.rs
