elusion 8.3.0

Elusion is a modern DataFrame / Data Engineering / Data Analysis library that combines the familiarity of DataFrame operations (like those in PySpark, Pandas, and Polars) with the power of SQL query building. It provides flexible query construction without enforcing strict operation ordering, enabling developers to write intuitive and maintainable data transformations.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

use crate::CustomDataFrame;
use crate::custom_error::cust_error::{ElusionError, ElusionResult};
use crate::project::config::project_config::{
    MaterializationType, OutputDestination, ProjectFile,
};
use super::local_writer::{write_parquet_local, write_delta_local};
use super::fabric_writer::write_parquet_fabric;

pub async fn materialize(
    df: &CustomDataFrame,
    model_name: &str,
    project_config: &ProjectFile,
) -> ElusionResult<()> {
    let mat_type = project_config.get_materialization(model_name);
    let output_path = project_config.get_output_path(model_name)?;

    // Memory = no write, just pass through
    if mat_type == MaterializationType::Memory {
        println!("💭 Memory only: '{}' not persisted", model_name);
        return Ok(());
    }

    match &project_config.output.destination {
        OutputDestination::Local => {
            match mat_type {
                MaterializationType::Parquet => {
                    write_parquet_local(df, &output_path, model_name).await
                }
                MaterializationType::Delta => {
                    write_delta_local(df, &output_path, model_name).await
                }
                MaterializationType::Memory => Ok(()),
            }
        }
        OutputDestination::Fabric => {
            let fabric = project_config.output.fabric.as_ref()
                .ok_or_else(|| ElusionError::Custom(
                    "❌ Fabric output not configured in elusion.toml".to_string()
                ))?;

            match mat_type {
                MaterializationType::Parquet | MaterializationType::Delta => {
                    write_parquet_fabric(
                        df,
                        &fabric.abfss_path,
                        &output_path,
                        &fabric.tenant_id,
                        &fabric.client_id,
                        &fabric.client_secret,
                        model_name,
                    ).await
                }
                MaterializationType::Memory => Ok(()),
            }
        }
    }
}