webdataset_core/
lib.rs

1//! Core data model for the [WebDataset](https://github.com/webdataset/webdataset) format.
2//!
3//! WebDataset stores a dataset as a set of POSIX tar archives ("shards"). The
4//! files inside a shard that share a basename make up one training sample:
5//!
6//! ```text
7//! shard-000000.tar
8//!   image0001.jpg   image0001.cls   image0001.json
9//!   image0002.jpg   image0002.cls   image0002.json
10//! ```
11//!
12//! This crate defines the pieces every other crate in the workspace shares:
13//!
14//! - [`Sample`] and [`Value`] — the in-memory shape of a training example.
15//! - [`Tensor`] and [`DType`] — dense numeric arrays, plus the [`npy`] codec.
16//! - [`Error`] and [`handlers`] — how failures are reported and absorbed.
17//! - [`braceexpand()`] and [`utils`] — shard-list expansion and worker identity.
18//!
19//! Most users should depend on the `webdataset` crate instead, which
20//! re-exports everything here.
21//!
22//! # `no_std`
23//!
24//! Turning off the `std` feature builds against `core` and `alloc` only, which
25//! covers the whole data model: samples, values, tensors, the `.npy` codec,
26//! brace expansion, and the error and handler types. What `std` adds is the
27//! parts that need an operating system — `Error::Io`, environment variable
28//! substitution, reading worker identity from the environment, and the
29//! stream-based `npy` helpers.
30//!
31//! ```toml
32//! webdataset-core = { version = "0.1", default-features = false, features = ["json"] }
33//! ```
34//!
35//! # Threads
36//!
37//! The `threads` feature — on by default — makes per-worker state thread-safe.
38//! With the standard library that is a thread-local and needs no setup. Without
39//! it, there is no portable way to ask which thread is running, so the host
40//! supplies one with
41//! [`workers::set_thread_id_hook`]; see the [`workers`]
42//! module. A single-threaded `no_std` program needs neither.
43
44#![doc(html_root_url = "https://docs.rs/webdataset-core/0.0.1")]
45#![cfg_attr(docsrs, feature(doc_cfg))]
46#![cfg_attr(not(feature = "std"), no_std)]
47
48extern crate alloc;
49
50/// The items the `std` prelude would provide, sourced from `alloc` so that the
51/// same code compiles with and without `std`.
52mod prelude {
53    #[allow(unused_imports)]
54    pub(crate) use alloc::borrow::ToOwned;
55    #[allow(unused_imports)]
56    pub(crate) use alloc::boxed::Box;
57    #[allow(unused_imports)]
58    pub(crate) use alloc::format;
59    #[allow(unused_imports)]
60    pub(crate) use alloc::string::{String, ToString};
61    #[allow(unused_imports)]
62    pub(crate) use alloc::vec;
63    #[allow(unused_imports)]
64    pub(crate) use alloc::vec::Vec;
65}
66
67pub mod braceexpand;
68pub mod error;
69pub mod fields;
70pub mod handlers;
71pub mod npy;
72pub mod sample;
73pub mod tensor;
74pub mod utils;
75pub mod value;
76pub mod workers;
77
78pub use braceexpand::{braceexpand, braceexpand_all};
79pub use error::{Error, Result, ResultExt};
80pub use fields::{FieldHasher, Fields};
81pub use handlers::{Action, Dispatch, Handler, HandlerRef};
82pub use sample::Sample;
83pub use tensor::{DType, Tensor};
84pub use utils::{WorkerInfo, base_plus_ext, expand_urls, worker_info};
85pub use value::{CustomValue, Value};
86pub use workers::{set_thread_id_hook, set_worker, with_worker};
webdataset_core/lib.rs

webdataset_core/
lib.rs
