delta_kernel 0.28.0

Core crate providing a Delta/Deltalake implementation focused on interoperability with a wide range of query engines.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

//! Interval-type integration tests for the CreateTable API.

use delta_kernel::committer::FileSystemCommitter;
use delta_kernel::expressions::column_name;
use delta_kernel::schema::{schema_ref, DataType};
use delta_kernel::transaction::create_table::create_table;
use delta_kernel::transaction::data_layout::DataLayout;
use delta_kernel::DeltaResult;
use test_utils::test_table_setup;

#[rstest::rstest]
fn test_create_table_rejects_interval_clustering(
    #[values(DataType::INTERVAL_YEAR_MONTH, DataType::INTERVAL_DAY_TIME)] interval: DataType,
    #[values(false, true)] nested: bool,
) -> DeltaResult<()> {
    let (_temp_dir, table_path, engine) = test_table_setup()?;
    let (schema, clustering_column) = if nested {
        (
            schema_ref! {
                not_null "id": INTEGER,
                nullable "nested": { nullable "iv": (interval) },
            },
            column_name!("nested.iv"),
        )
    } else {
        (
            schema_ref! {
                not_null "id": INTEGER,
                nullable "iv": (interval),
            },
            column_name!("iv"),
        )
    };

    let result = create_table(&table_path, schema, "Test/1.0")
        .with_data_layout(DataLayout::Clustered {
            columns: vec![clustering_column],
        })
        .build(engine.as_ref(), Box::new(FileSystemCommitter::new()));
    test_utils::assert_result_error_with_message(result, "unsupported type");
    Ok(())
}

mod supported {
    use delta_kernel::schema::SchemaRef;
    use delta_kernel::snapshot::Snapshot;
    use delta_kernel::table_features::ColumnMappingMode;
    use rstest::rstest;
    use test_utils::cm_properties;

    use super::super::column_mapping::{
        assert_column_mapping_config, strip_column_mapping_metadata,
    };
    use super::*;

    /// Top-level schema carrying the given interval `DataType`.
    fn top_level_interval_schema(interval: DataType) -> SchemaRef {
        schema_ref! {
            not_null "id": INTEGER,
            nullable "iv": (interval),
        }
    }

    /// Schema with the given interval `DataType` nested inside a struct.
    fn nested_interval_schema(interval: DataType) -> SchemaRef {
        schema_ref! {
            not_null "id": INTEGER,
            nullable "nested": {
                nullable "inner_iv": (interval),
            },
        }
    }

    /// Creating a table with interval columns preserves its schema across column mapping modes.
    #[rstest]
    fn test_create_table_with_interval_round_trips_schema(
        #[values(DataType::INTERVAL_YEAR_MONTH, DataType::INTERVAL_DAY_TIME)] interval: DataType,
        #[values(top_level_interval_schema, nested_interval_schema)] make_schema: fn(
            DataType,
        )
            -> SchemaRef,
        #[values("none", "name", "id")] cm_mode: &str,
    ) -> DeltaResult<()> {
        let (_temp_dir, table_path, engine) = test_table_setup()?;
        let schema = make_schema(interval);

        let _ = create_table(&table_path, schema.clone(), "Test/1.0")
            .with_table_properties(cm_properties(cm_mode))
            .build(engine.as_ref(), Box::new(FileSystemCommitter::new()))?
            .commit(engine.as_ref())?;

        let table_url = delta_kernel::try_parse_uri(&table_path)?;
        let snapshot = Snapshot::builder_for(table_url).build(engine.as_ref())?;

        let expected_cm_mode = match cm_mode {
            "none" => ColumnMappingMode::None,
            "name" => ColumnMappingMode::Name,
            "id" => ColumnMappingMode::Id,
            _ => unreachable!(),
        };
        assert_column_mapping_config(&snapshot, expected_cm_mode);

        let read_schema = snapshot.schema();
        let stripped_schema = strip_column_mapping_metadata(&read_schema);
        assert_eq!(
            &stripped_schema,
            schema.as_ref(),
            "logical schema should round-trip through create table"
        );
        Ok(())
    }
}