delta_kernel 0.6.0

Core crate providing a Delta/Deltalake implementation focused on interoperability with a wide range of query engines.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

//! Schema definitions for action types

use std::collections::{HashMap, HashSet};

use crate::schema::{ArrayType, DataType, MapType, StructField, StructType};

pub(crate) trait ToSchema {
    fn to_schema() -> StructType;
}

pub(crate) trait ToDataType {
    fn to_data_type() -> DataType;
}

impl<T: ToSchema> ToDataType for T {
    fn to_data_type() -> DataType {
        T::to_schema().into()
    }
}

pub(crate) trait ToNullableContainerType {
    fn to_nullable_container_type() -> DataType;
}

macro_rules! impl_to_data_type {
    ( $(($rust_type: ty, $kernel_type: expr)), * ) => {
        $(
            impl ToDataType for $rust_type {
                fn to_data_type() -> DataType {
                    $kernel_type
                }
            }
        )*
    };
}

impl_to_data_type!(
    (String, DataType::STRING),
    (i64, DataType::LONG),
    (i32, DataType::INTEGER),
    (i16, DataType::SHORT),
    (char, DataType::BYTE),
    (f32, DataType::FLOAT),
    (f64, DataType::DOUBLE),
    (bool, DataType::BOOLEAN)
);

// ToDataType impl for non-nullable array types
impl<T: ToDataType> ToDataType for Vec<T> {
    fn to_data_type() -> DataType {
        ArrayType::new(T::to_data_type(), false).into()
    }
}

impl<T: ToDataType> ToDataType for HashSet<T> {
    fn to_data_type() -> DataType {
        ArrayType::new(T::to_data_type(), false).into()
    }
}

// ToDataType impl for non-nullable map types
impl<K: ToDataType, V: ToDataType> ToDataType for HashMap<K, V> {
    fn to_data_type() -> DataType {
        MapType::new(K::to_data_type(), V::to_data_type(), false).into()
    }
}

// ToDataType impl for nullable map types
impl<K: ToDataType, V: ToDataType> ToNullableContainerType for HashMap<K, V> {
    fn to_nullable_container_type() -> DataType {
        MapType::new(K::to_data_type(), V::to_data_type(), true).into()
    }
}

pub(crate) trait GetStructField {
    fn get_struct_field(name: impl Into<String>) -> StructField;
}

pub(crate) trait GetNullableContainerStructField {
    fn get_nullable_container_struct_field(name: impl Into<String>) -> StructField;
}

// Normal types produce non-nullable fields, but in this case the container they reference has
// nullable values
impl<T: ToNullableContainerType> GetNullableContainerStructField for T {
    fn get_nullable_container_struct_field(name: impl Into<String>) -> StructField {
        StructField::new(name, T::to_nullable_container_type(), false)
    }
}

// Normal types produce non-nullable fields
impl<T: ToDataType> GetStructField for T {
    fn get_struct_field(name: impl Into<String>) -> StructField {
        StructField::new(name, T::to_data_type(), false)
    }
}

// Option types produce nullable fields
impl<T: ToDataType> GetStructField for Option<T> {
    fn get_struct_field(name: impl Into<String>) -> StructField {
        StructField::new(name, T::to_data_type(), true)
    }
}