delta-sharing 0.2.0

Delta Sharing client library
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152

//! Delta Sharing Protocol message types

use serde::{Deserialize, Serialize};
use serde_json::value::Value;
use serde_json::Map;

#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ProviderConfig {
    pub share_credentials_version: i32,
    pub endpoint: String,
    pub bearer_token: String,
}

#[derive(Deserialize, Debug, Clone)]
pub struct Share {
    pub name: String,
}

#[derive(Deserialize, Debug, Clone)]
pub struct Schema {
    pub name: String,
    pub share: String,
}

#[derive(Deserialize, Debug, Clone)]
pub struct Table {
    pub name: String,
    pub share: String,
    pub schema: String,
}

impl Table {
    pub fn fully_qualified_name(&self) -> String {
        format!("{}.{}.{}", self.share, self.schema, self.name)
    }
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DeltaProtocol {
    pub min_reader_version: i32,
    pub min_writer_version: i32,
    pub reader_features: Vec<String>,
    pub writer_features: Vec<String>,
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
#[serde(untagged)]
pub enum Protocol {
    Parquet {
        #[serde(rename = "minReaderVersion")]
        min_reader_version: i32,
    },
    Delta {
        #[serde(rename = "deltaProtocol")]
        delta_protocol: DeltaProtocol,
    },
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
pub struct Format {
    pub provider: String,
    pub options: Option<Map<String, Value>>,
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ParquetMetadata {
    pub id: String,
    pub name: Option<String>,
    pub description: Option<String>,
    pub format: Format,
    pub schema_string: String,
    pub configuration: Map<String, Value>,
    pub partition_columns: Vec<String>,
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DeltaMetadata {
    pub size: usize,
    pub num_files: usize,
    pub delta_metadata: ParquetMetadata,
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
#[serde(untagged)]
pub enum Metadata {
    Parquet(ParquetMetadata),
    Delta(DeltaMetadata),
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
pub struct TableMetadata {
    pub protocol: Protocol,
    pub metadata: Metadata,
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ParquetFile {
    pub id: String,
    pub url: String,
    pub partition_values: Map<String, Value>,
    pub size: i32,
    pub stats: Option<String>,
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DeltaFile {
    pub id: String,
    pub deletion_vector_file_id: Option<String>,
    pub version: Option<usize>,
    pub timestamp: Option<usize>,
    pub expiration_timestamp: Option<usize>,
    pub delta_single_action: Map<String, Value>,
}

impl DeltaFile {
    pub fn get_url(&self) -> Option<String> {
        if let Some(value) = self
            .delta_single_action
            .get("add")
            .and_then(|add| add.get("path"))
        {
            return value.as_str().map(|v| v.to_string());
        } else {
            return None;
        }
    }
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
#[serde(untagged)]
pub enum File {
    Parquet(ParquetFile),
    Delta(DeltaFile),
}

#[derive(Deserialize, Debug, Clone, PartialEq, Serialize)]
pub struct TableFiles {
    pub metadata: TableMetadata,
    pub files: Vec<File>,
}

pub struct FilesRequest {
    pub predicate_hints: Option<Vec<String>>,
    pub limit_hint: Option<i32>,
    pub version: Option<i32>,
}