[package]
edition = "2024"
rust-version = "1.85"
name = "webdataset"
version = "0.0.1"
authors = ["Eugene Hauptmann"]
build = false
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "High-performance sequential dataset loading from tar archives, in the WebDataset format"
homepage = "https://github.com/eugenehp/webdataset-rs"
readme = "README.md"
keywords = [
"webdataset",
"tar",
"dataset",
"machine-learning",
"dataloader",
]
categories = [
"science",
"data-structures",
"encoding",
]
license = "BSD-3-Clause"
repository = "https://github.com/eugenehp/webdataset-rs"
[package.metadata.docs.rs]
features = ["full"]
rustdoc-args = [
"--cfg",
"docsrs",
]
[features]
async = [
"dep:futures-core",
"dep:futures-io",
"dep:futures-util",
"webdataset-shard/async",
]
bzip2 = ["webdataset-shard/bzip2"]
cbor = ["dep:ciborium"]
default = [
"yaml",
"threads",
"subprocess",
]
full = [
"image",
"msgpack",
"cbor",
"npz",
"yaml",
"threads",
"subprocess",
"async",
"zstd",
"bzip2",
"xz",
]
image = [
"dep:image",
"webdataset-core/image",
]
libjpeg = [
"dep:turbojpeg",
"image",
]
msgpack = ["dep:rmp-serde"]
npz = ["dep:zip"]
subprocess = [
"webdataset-io/subprocess",
"webdataset-shard/subprocess",
]
threads = [
"webdataset-core/threads",
"webdataset-shard/threads",
]
wasm-js = [
"dep:getrandom",
"getrandom/wasm_js",
]
xz = ["webdataset-shard/xz"]
yaml = ["dep:serde_yaml"]
zstd = ["webdataset-shard/zstd"]
[lib]
name = "webdataset"
path = "src/lib.rs"
[[example]]
name = "async_pipeline"
path = "examples/async_pipeline.rs"
required-features = ["async"]
[[example]]
name = "prelude_check"
path = "examples/prelude_check.rs"
[[example]]
name = "read_dataset"
path = "examples/read_dataset.rs"
[[example]]
name = "training_loop"
path = "examples/training_loop.rs"
[[example]]
name = "write_dataset"
path = "examples/write_dataset.rs"
[[test]]
name = "async_equivalence"
path = "tests/async_equivalence.rs"
[[test]]
name = "portable"
path = "tests/portable.rs"
[[test]]
name = "reference_data"
path = "tests/reference_data.rs"
[[test]]
name = "roundtrip"
path = "tests/roundtrip.rs"
[[bench]]
name = "pipeline"
path = "benches/pipeline.rs"
harness = false
[dependencies.bytes]
version = "1.12"
features = ["std"]
default-features = false
[dependencies.ciborium]
version = "0.2"
optional = true
[dependencies.flate2]
version = "1.1"
[dependencies.futures-core]
version = "0.3"
features = ["std"]
optional = true
default-features = false
[dependencies.futures-io]
version = "0.3"
optional = true
[dependencies.futures-util]
version = "0.3"
features = [
"std",
"io",
]
optional = true
default-features = false
[dependencies.glob]
version = "0.3"
[dependencies.image]
version = "0.25"
optional = true
[dependencies.indexmap]
version = "2.14"
features = ["std"]
default-features = false
[dependencies.log]
version = "0.4"
[dependencies.rand]
version = "0.10"
[dependencies.regex]
version = "1.13"
[dependencies.rmp-serde]
version = "1.3"
optional = true
[dependencies.serde_json]
version = "1.0"
features = ["std"]
default-features = false
[dependencies.serde_yaml]
version = "0.9"
optional = true
[dependencies.turbojpeg]
version = "1.5"
features = [
"pkg-config",
"cmake",
]
optional = true
default-features = false
[dependencies.webdataset-core]
version = "0.0.1"
features = [
"std",
"json",
]
default-features = false
[dependencies.webdataset-io]
version = "0.0.1"
default-features = false
[dependencies.webdataset-shard]
version = "0.0.1"
default-features = false
[dependencies.webdataset-tenbin]
version = "0.0.1"
features = ["std"]
default-features = false
[dependencies.zip]
version = "7.1"
features = ["deflate"]
optional = true
default-features = false
[dev-dependencies.criterion]
version = "0.7"
features = ["cargo_bench_support"]
default-features = false
[dev-dependencies.futures-channel]
version = "0.3"
[dev-dependencies.futures-executor]
version = "0.3"
[dev-dependencies.tempfile]
version = "3.27"
[target.'cfg(target_arch = "wasm32")'.dependencies.getrandom]
version = "0.4"
optional = true
[lints.rust]
missing_docs = "warn"
unsafe_code = "forbid"
[lints.rust.unexpected_cfgs]
level = "warn"
priority = 0
check-cfg = ["cfg(docsrs)"]