floe-core 0.3.6

Core library for Floe, a YAML-driven technical ingestion tool.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

use polars::prelude::DataFrame;

use crate::errors::RunError;
use crate::io::format::AcceptedMergeMetrics;
use crate::io::storage::Target;
use crate::{config, FloeResult};

pub(crate) mod scd1;
pub(crate) mod scd2;
pub(crate) mod shared;

pub(crate) struct MergeExecutionContext<'a> {
    pub(crate) runtime: &'a tokio::runtime::Runtime,
    pub(crate) target: &'a Target,
    pub(crate) resolver: &'a config::StorageResolver,
    pub(crate) entity: &'a config::EntityConfig,
    pub(crate) partition_by: Option<Vec<String>>,
    pub(crate) target_file_size_bytes: Option<usize>,
}

/// MergeBackend separates format-specific merge primitives (Delta today, others later)
/// from merge strategy orchestration so new table formats can plug in without duplicating
/// key resolution, validation, and reporting flow.
pub(crate) trait MergeBackend {
    fn execute_scd1(
        &self,
        source_df: &mut DataFrame,
        ctx: &MergeExecutionContext<'_>,
    ) -> FloeResult<(
        i64,
        AcceptedMergeMetrics,
        crate::io::format::AcceptedSchemaEvolution,
        shared::DeltaMergePerfBreakdown,
    )>;

    fn execute_scd2(
        &self,
        _source_df: &mut DataFrame,
        _ctx: &MergeExecutionContext<'_>,
    ) -> FloeResult<(
        i64,
        AcceptedMergeMetrics,
        crate::io::format::AcceptedSchemaEvolution,
        shared::DeltaMergePerfBreakdown,
    )> {
        Err(Box::new(RunError(
            "write_mode=merge_scd2 is not implemented for this backend".to_string(),
        )))
    }
}