hyperfile 0.3.2

A file-like interface abstraction layer provides random read and write capabilities for S3
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

[package]
name = "hyperfile"
version = "0.3.2"
edition = "2024"
rust-version = "1.85"
description = "A file-like interface abstraction layer provides random read and write capabilities for S3"
authors = ["Dai Yiyang"]
license = "Apache-2.0"
categories = ["asynchronous", "filesystem", "data-structures"]
keywords = ["file", "object-storage", "s3"]
homepage = "https://github.com/hyperfile/hyperfile"
repository = "https://github.com/hyperfile/hyperfile"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
log = "0.4.26"
env_logger = "0.11.6"
serde = { version = "1.0.218", features = ["derive"] }
tokio = { version = "1", features = ["full"] }
libc = "0.2.170"
chrono = "0.4.39"
bytes = "1.10.0"
http-body = "0.4"
http = "0.2"
aws-smithy-types = "1"
lru = "0.14.0"
rand = { version = "0.9.0", optional = true }
rangemap = { version = "1.5.1", optional = true }
aws-config = { version = "1.5.17", features = ["behavior-version-latest"] }
aws-sdk-s3 = "1.77.0"
futures = "0.3.31"
ulid = "1.2.1"
foyer = "0.19.2"
serde_json = "1.0.145"
btree-ondisk = { version = "0.18.0", default-features = false, features = ["tokio-runtime"] }
hyperfile-reactor = { version = "0.3.2", optional = true }

[dev-dependencies]
rand = "0.9.0"
md5 = "0.7.0"
crc64fast = "1.1.0"
human_bytes = "0.4.3"

[features]
# --------------------------------------------------------------------
# Default features
# --------------------------------------------------------------------

# `reactor` + `meta_loader_batch` is the recommended setup for new
# users. It gives you the async request/response handle API
# (HyperFileHandler / HyperFileTokio) and the batched meta-block
# loader that cuts down on S3 round-trips during reads.
default = ["reactor", "meta_loader_batch"]

# --------------------------------------------------------------------
# Optional features
# --------------------------------------------------------------------

# Enable the batched meta-block loader (`src/meta_loader/s3_batch.rs`),
# which coalesces multiple meta-block reads into a single S3 request
# chain. Without this feature the per-block loader at
# `src/meta_loader/s3.rs` is used, issuing one S3 GET per meta block.
#
# Recommended: keep enabled. Disabling trades throughput on read for
# a simpler call path.
meta_loader_batch = []

# Enable write-ahead logging. Writes are durably persisted to a
# separate S3 prefix (configured via `HyperFileWalConfig`) before
# they are acknowledged. On reopen, unflushed writes are replayed.
#
# Cost: every write incurs an extra S3 PUT to the WAL prefix. Every
# successful flush removes the corresponding WAL chunks.
#
# Without this feature, writes sit in memory only until flush; a
# crash between write and flush loses the unflushed data. With this
# feature the file stays consistent across crashes up to the last
# write acknowledged.
wal = []

# Enable optimistic fine-grained byte-range locking inside a single
# process. Concurrent writes to disjoint byte ranges run in parallel;
# overlapping writes serialize. Required for the range-lock behavior
# in HyperFileHandler when multiple clones race on the same file.
#
# Cross-process concurrency is still governed by the conflict policy
# (HyperFileConfig::flush_conflict_policy), not by this feature.
#
# See docs/concurrency.md for the full model.
range-lock = ["dep:rangemap"]

# Parallelize segment payload construction during flush. Effective
# **only when** the `wal` feature is also enabled — the WAL feature
# pins the in-memory segment as a single contiguous buffer and the
# concurrent build memcpys dirty data blocks into it via
# tokio::spawn_blocking, which is what this feature parallelizes.
#
# Without `wal`, the segment body is built as a list of `Bytes`
# pieces (no contiguous buffer to concat into), so this feature is
# effectively a no-op on the non-WAL path. Phase 1 of the zero-
# copy / scatter-gather work made the non-WAL path memory-light
# and removed the only memcpy this feature was useful against.
#
# Caveats (WAL path only):
#   - Adds a spin-wait on the flush hot path; see the NOTE in
#     `src/file/mod.rs::flush_process_build_segment` for the
#     tradeoffs.
#   - Benchmarks are not yet in the repository; the break-even
#     point between serial and parallel construction has not
#     been measured.
concurrent-segment-build = []

# --------------------------------------------------------------------
# Runtime-model features (mutually exclusive)
# --------------------------------------------------------------------
#
# Exactly one of `reactor` or `blocking` should be enabled at a time.
# Both `btree-ondisk` and `src/lib.rs` contain `compile_error!`
# guards for the forbidden combination: `btree-ondisk` picks between
# `arc` (reactor) and `rc` (blocking) for its internal
# reference-counted nodes, and the two cannot coexist.

# Reactor mode. Wraps the file in an event-driven handler task (see
# `src/file/handler.rs`): a dedicated OS thread with a current-thread
# tokio runtime serializes all file operations through a request /
# response channel. Reduces contention under concurrent access to the
# same file.
#
# Use the `fh_*` family in `src/file/fh.rs` (HyperFileHandler /
# HyperFileTokio) from application code; do not call `fs_*` directly
# on the inner Hyper. Selected by the default feature set.
reactor = ["dep:hyperfile-reactor", "btree-ondisk/arc", "btree-ondisk/value-check", "btree-ondisk/mt" ]

# Blocking mode. The general-purpose single-task model: call the
# `fs_*` family in `src/file/fs.rs` directly on the file and provide
# your own concurrency control. Uses Rc instead of Arc for
# btree-ondisk nodes, avoiding some atomics.
#
# Do not enable together with `reactor`.
blocking = ["btree-ondisk/rc"]

# --------------------------------------------------------------------
# Development / internal features
# --------------------------------------------------------------------

# Compile in the staging benchmarking helpers under
# `src/staging/bench.rs`. Not intended for production use; pulls in
# `rand` as a non-dev dependency.
bench = ["dep:rand"]