delta_kernel 0.19.1

Core crate providing a Delta/Deltalake implementation focused on interoperability with a wide range of query engines.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158

[package]
name = "delta_kernel"
description = "Core crate providing a Delta/Deltalake implementation focused on interoperability with a wide range of query engines."
documentation = "https://docs.rs/delta_kernel"
edition.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
readme.workspace = true
version.workspace = true
# exclude golden tests + golden test data since they push us over 10MB crate size limit
# also exlude tests/data since similarly this has large test tables we don't need to ship
exclude = ["tests/golden_tables.rs", "tests/golden_data/", "tests/data/"]
rust-version.workspace = true

[package.metadata.docs.rs]
all-features = true

# for cargo-release
[package.metadata.release]
pre-release-replacements = [
  { file = "../README.md", search = "delta_kernel = \"[a-z0-9\\.-]+\"", replace = "delta_kernel = \"{{version}}\"" },
  { file = "../README.md", search = "version = \"[a-z0-9\\.-]+\"", replace = "version = \"{{version}}\"" },
]
pre-release-hook = [
  "git",
  "cliff",
  "--repository",
  "../",
  "--config",
  "../cliff.toml",
  "--unreleased",
  "--prepend",
  "../CHANGELOG.md",
  "--include-path",
  "*",
  "--tag",
  "{{version}}",
]

[dependencies]
delta_kernel_derive = { path = "../derive-macros", version = "0.19.0" }
bytes = "1.10"
chrono = "0.4.41"
crc = "3.2.2"
indexmap = "2.10.0"
itertools = "0.14"
roaring = "0.11.2"
serde = { version = "1", features = ["derive", "rc"] }
serde_json = "1"
strum = { version = "0.27", features = ["derive"] }
thiserror = "2"
# only for structured logging
tracing = { version = "0.1", features = ["log"] }
url = "2"
uuid = { version = "1.18.0", features = ["v4", "fast-rng"] }
z85 = "3.0.6"

# optional deps
futures = { version = "0.3", optional = true }
# Used for fetching direct urls (like pre-signed urls)
reqwest = { version = "0.12.23", default-features = false, optional = true }
# optionally used with default engine (though not required)
tokio = { version = "1.47", optional = true, features = ["rt-multi-thread"] }
# both arrow versions below are optional and require object_store
object_store = { version = "0.12.3", optional = true, features = ["aws", "azure", "gcp", "http"] }
comfy-table = { version = "7.1", optional = true }

# arrow 56
[dependencies.arrow_56]
package = "arrow"
version = "56"
features = ["chrono-tz", "ffi", "json", "prettyprint"]
optional = true
[dependencies.parquet_56]
package = "parquet"
version = "56"
features = ["async", "object_store"]
optional = true

# arrow 57
[dependencies.arrow_57]
package = "arrow"
version = "57"
features = ["chrono-tz", "ffi", "json", "prettyprint"]
optional = true
[dependencies.parquet_57]
package = "parquet"
version = "57"
features = ["async", "object_store"]
optional = true

[features]
# no default features
default = []
# internal-api will make everything marked #[internal_api] public
internal-api = []
# integration-test turns on a particularly heavy test for hdfs-object-store
integration-test = ["hdfs-native-object-store/integration-test"]

# The default versions for arrow/parquet/object_store
arrow = ["arrow-57"] # latest arrow version
need-arrow = [] # need-arrow is a marker that the feature needs arrow dep

arrow-56 = ["dep:arrow_56", "dep:parquet_56", "object_store", "comfy-table"]
arrow-57 = ["dep:arrow_57", "dep:parquet_57", "object_store", "comfy-table"]
arrow-conversion = ["need-arrow"]
arrow-expression = ["need-arrow"]

# WARNING: experimental feature, still under active development
# enables new experimental catalog-managed tables support
catalog-managed = []

# this is an 'internal' feature flag which has all the shared bits from default-engine and
# default-engine-rustls
default-engine-base = [
  "arrow-conversion",
  "arrow-expression",
  "futures",
  "need-arrow",
  "tokio",
]
# the default-engine-native-tls use the reqwest crate with default features which uses native-tls. if you want
# to instead use rustls, use 'default-engine-rustls' which has no native-tls dependency
default-engine-native-tls = ["default-engine-base", "reqwest/default"]
default-engine-rustls = [
  "default-engine-base",
  "reqwest/rustls-tls-native-roots",
  "reqwest/http2",
]

[build-dependencies]
rustc_version = "0.4.1"

[dev-dependencies]
delta_kernel = { path = ".", features = ["arrow", "catalog-managed", "default-engine-rustls", "internal-api"] }
test_utils = { path = "../test-utils" }
criterion = "0.5"
# Used for testing parse_url_opts extensibility
hdfs-native-object-store = { version = "0.15.0" }
hdfs-native = "0.12.2"
walkdir = { version = "2.5.0" }
async-trait = "0.1" # only used for our custom SlowGetStore ObjectStore implementation
paste = "1.0"
test-log = { version = "0.2", default-features = false, features = ["trace"] }
tempfile = "3"
tracing-subscriber = { version = "0.3", default-features = false, features = [
  "env-filter",
  "fmt",
] }

[[bench]]
name = "metadata_bench"
harness = false

[[bench]]
name = "expression_bench"
harness = false