datafusion-distributed 4.0.0

Framework for enhancing Apache DataFusion with distributed capabilities
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

[workspace]
members = [
    "benchmarks",
    "cli",
    "console",
]

[workspace.dependencies]
datafusion = { version = "55", default-features = false }
datafusion-proto = { version = "55", default-features = false }

[package]
name = "datafusion-distributed"
version = "4.0.0"
edition = "2024"
description = "Framework for enhancing Apache DataFusion with distributed capabilities"
license = "Apache-2.0"
documentation = "https://datafusion-contrib.github.io/datafusion-distributed/"
repository = "https://github.com/datafusion-contrib/datafusion-distributed"

[dependencies]
chrono = { version = "0.4.44" }
datafusion = { workspace = true, features = [
  "unicode_expressions",
  "datetime_expressions",
] }
datafusion-proto = { workspace = true }
async-trait = "0.1.89"
tokio = { version = "1.48", features = ["full"] }
http = "1.3.1"
itertools = "0.14.0"
futures = "0.3.31"
url = "2.5.7"
uuid = "1.19"
delegate = "0.13.4"
dashmap = "6.0.1"
prost = "0.14.1"
rand = "0.9"
object_store = "0.13"
bytes = "1.11"
pin-project = "1.1.10"
tokio-stream = { version = "0.1.17", features = ["sync"] }
tokio-util = "0.7"
moka = { version = "0.12", features = ["sync", "future"] }
crossbeam-queue = "0.3"
sysinfo = { version = "0.30", optional = true }
sketches-ddsketch = { version = "0.3", features = ["use_serde"] }
num-traits = "0.2"
bincode = "1"
tonic-prost = "0.14.2"

# grpc-specific features
arrow-flight = { version = "59", optional = true }
arrow-select = { version = "59", optional = true }
arrow-ipc = { version = "59", features = ["zstd"], optional = true }
tonic = { version = "0.14.1", features = ["transport"], optional = true }
tower = { version = "0.5.2", optional = true }

# integration_tests deps
insta = { version = "1.46.0", features = ["filters"], optional = true }
parquet = { version = "59", optional = true }
arrow = { version = "59", optional = true, features = ["test_utils"] }
hyper-util = { version = "0.1.16", optional = true }

[features]
default = ["grpc", "parquet", "sql"]

# Both are in the default set, so nothing changes for consumers who do not opt out.
# They exist so that an embedder which never reads parquet files and never parses SQL
# can skip the parquet and sqlparser stacks with `default-features = false`.
parquet = ["datafusion/parquet", "datafusion-proto/parquet"]
sql = ["datafusion/sql"]
avro = ["datafusion/avro"]
grpc = [
    "arrow-flight",
    "arrow-select",
    "arrow-ipc",
    "tonic",
    "tower"
]

integration = ["insta", "dep:parquet", "arrow", "hyper-util", "grpc", "parquet", "sql"]

system-metrics = ["sysinfo"]

tpch = ["integration"]
tpcds = ["integration"]
clickbench = ["integration"]
slow-tests = []
sysinfo = ["dep:sysinfo"]

[dev-dependencies]
datafusion-distributed-benchmarks = { path = "benchmarks" }
# The crate's own tests drive `SessionContext::sql` and register parquet tables, so they need
# both features regardless of which ones a consumer selects. Requesting them here keeps
# `--no-default-features` test runs compiling without forcing either on library consumers.
datafusion = { workspace = true, features = ["parquet", "sql"] }
datafusion-proto = { workspace = true, features = ["parquet"] }
structopt = "0.3"
insta = { version = "1.46.0", features = ["filters"] }
parquet = "59"
arrow = { version = "59", features = ["test_utils"] }
tokio-stream = { version = "0.1.17", features = ["sync"] }
hyper-util = "0.1.16"
pretty_assertions = "1.4"
test-case = "3.3.1"

[workspace.lints.clippy]
disallowed_types = "deny"
disallowed-methods = "deny"