floe-core 0.3.1

Core library for Floe, a YAML-driven technical ingestion tool.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

use std::path::Path;

use polars::prelude::DataFrame;

use crate::io::format::{AcceptedSinkAdapter, AcceptedWriteOutput};
use crate::io::storage::Target;
use crate::{config, io, FloeResult};

pub struct AcceptedWriteRequest<'a> {
    pub target: &'a Target,
    pub df: &'a mut DataFrame,
    pub output_stem: &'a str,
    pub temp_dir: Option<&'a Path>,
    pub cloud: &'a mut io::storage::CloudClient,
    pub resolver: &'a config::StorageResolver,
    pub catalogs: &'a config::CatalogResolver,
    pub entity: &'a config::EntityConfig,
    pub mode: config::WriteMode,
}

pub trait AcceptedWriter: Send + Sync {
    fn write(&self, request: AcceptedWriteRequest<'_>) -> FloeResult<AcceptedWriteOutput>;
}

pub struct AdapterAcceptedWriter<'a> {
    adapter: &'a dyn AcceptedSinkAdapter,
}

impl<'a> AdapterAcceptedWriter<'a> {
    pub fn new(adapter: &'a dyn AcceptedSinkAdapter) -> Self {
        Self { adapter }
    }
}

impl AcceptedWriter for AdapterAcceptedWriter<'_> {
    fn write(&self, request: AcceptedWriteRequest<'_>) -> FloeResult<AcceptedWriteOutput> {
        self.adapter.write_accepted(
            request.target,
            request.df,
            request.mode,
            request.output_stem,
            request.temp_dir,
            request.cloud,
            request.resolver,
            request.catalogs,
            request.entity,
        )
    }
}

pub fn write_with_adapter(
    adapter: &dyn AcceptedSinkAdapter,
    request: AcceptedWriteRequest<'_>,
) -> FloeResult<AcceptedWriteOutput> {
    AdapterAcceptedWriter::new(adapter).write(request)
}