datafusion-distributed 2.0.0

Framework for enhancing Apache DataFusion with distributed capabilities
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

[workspace]
members = ["benchmarks", "cli", "console"]

[workspace.dependencies]
datafusion = { version = "54.0.0", default-features = false }
datafusion-proto = { version = "54.0.0" }

[package]
name = "datafusion-distributed"
version = "2.0.0"
edition = "2024"
description = "Framework for enhancing Apache DataFusion with distributed capabilities"
license = "Apache-2.0"
documentation = "https://datafusion-contrib.github.io/datafusion-distributed/"
repository = "https://github.com/datafusion-contrib/datafusion-distributed"

[dependencies]
chrono = { version = "0.4.44" }
datafusion = { workspace = true, features = [
  "parquet",
  "sql",
  "unicode_expressions",
  "datetime_expressions",
] }
datafusion-proto = { workspace = true }
arrow-flight = "58"
arrow-select = "58"
arrow-ipc = { version = "58", features = ["zstd"] }
async-trait = "0.1.89"
tokio = { version = "1.48", features = ["full"] }
tonic = { version = "0.14.1", features = ["transport"] }
tower = "0.5.2"
http = "1.3.1"
itertools = "0.14.0"
futures = "0.3.31"
url = "2.5.7"
uuid = "1.19"
delegate = "0.13.4"
dashmap = "6.0.1"
prost = "0.14.1"
rand = "0.9"
object_store = "0.13"
bytes = "1.11"
pin-project = "1.1.10"
tokio-stream = { version = "0.1.17", features = ["sync"] }
tokio-util = "0.7"
moka = { version = "0.12", features = ["sync", "future"] }
crossbeam-queue = "0.3"
sysinfo = { version = "0.30", optional = true }
sketches-ddsketch = { version = "0.3", features = ["use_serde"] }
bincode = "1"
tonic-prost = "0.14.2"

# integration_tests deps
insta = { version = "1.46.0", features = ["filters"], optional = true }
parquet = { version = "58", optional = true }
arrow = { version = "58", optional = true, features = ["test_utils"] }
hyper-util = { version = "0.1.16", optional = true }

[features]
avro = ["datafusion/avro"]
integration = ["insta", "parquet", "arrow", "hyper-util"]

system-metrics = ["sysinfo"]

tpch = ["integration"]
tpcds = ["integration"]
clickbench = ["integration"]
slow-tests = []
sysinfo = ["dep:sysinfo"]

[dev-dependencies]
datafusion-distributed-benchmarks = { path = "benchmarks" }
structopt = "0.3"
insta = { version = "1.46.0", features = ["filters"] }
parquet = "58"
arrow = { version = "58", features = ["test_utils"] }
tokio-stream = { version = "0.1.17", features = ["sync"] }
hyper-util = "0.1.16"
pretty_assertions = "1.4"
test-case = "3.3.1"

[workspace.lints.clippy]
disallowed_types = "deny"
disallowed-methods = "deny"