Skip to main content

incin_data/
lib.rs

1//! # Incin Data
2//!
3//! Provides dataset and dataloader abstractions for the Incin framework.
4//! Handles batching, shuffling, and data transformation pipelines.
5//!
6//! ## Examples
7//!
8//! A `Dataset` supplies items by index; a `Collate` can turn a batch into the
9//! model's input type; a `DataLoader` joins the two and yields typed `Result`
10//! values so worker failures cannot look like end-of-data. Use
11//! `DataLoader::builder` for the default scalar, tuple, and tensor batching
12//! policy.
13//!
14//! ```rust
15//! use incin_data::{Collate, DataLoader, Dataset};
16//!
17//! struct Squares;
18//!
19//! impl Dataset for Squares {
20//!     type Item = u32;
21//!
22//!     fn len(&self) -> usize {
23//!         100
24//!     }
25//!
26//!     fn get(&self, index: usize) -> Result<Option<Self::Item>, incin_data::DataError> {
27//!         Ok(u32::try_from(index).ok().filter(|_| index < self.len()).map(|i| i * i))
28//!     }
29//! }
30//!
31//! struct IntoBatch;
32//!
33//! impl Collate<u32> for IntoBatch {
34//!     type Output = Vec<u32>;
35//!
36//!     fn collate(&self, batch: Vec<u32>) -> incin_data::BatchResult<Self::Output> {
37//!         Ok(batch)
38//!     }
39//! }
40//!
41//! // Batches of 32, shuffled. Loading happens on worker threads; iterating
42//! // borrows the loader, so the same one can be iterated each epoch.
43//! let loader = DataLoader::new(Squares, IntoBatch, 32).unwrap().with_shuffle(true);
44//!
45//! let batches: Vec<Vec<u32>> = (&loader).into_iter().collect::<Result<_, _>>().unwrap();
46//! assert_eq!(batches.len(), 4); // 100 items is three full batches and a short one
47//! assert_eq!(batches.iter().map(Vec::len).sum::<usize>(), 100);
48//! ```
49
50#[macro_use]
51extern crate alloc;
52/// Dataset.
53pub mod dataset;
54/// Downloader.
55#[cfg(feature = "download")]
56pub mod downloader;
57/// Hub.
58#[cfg(feature = "hub")]
59pub mod hub;
60/// Loader.
61pub mod loader;
62/// Transforms and data processing pipelines.
63pub mod transforms;
64/// Vision.
65pub mod vision;
66
67pub use dataset::Dataset;
68#[cfg(feature = "download")]
69pub use downloader::Downloader;
70pub use loader::{BatchResult, Collate, DataError, DataLoader, DataLoaderBuilder, DefaultCollate};
71pub use transforms::{CenterCrop, Compose, Normalize, RandomHorizontalFlip, Scale, Transform};
72
73/// Prelude.
74pub mod prelude {
75    #[cfg(feature = "hub")]
76    pub use super::hub::{HubApi, HubRepo, download, from_pretrained};
77    pub use super::loader::{
78        BatchResult, Collate, DataError, DataLoader, DataLoaderBuilder, DataLoaderIter,
79        DefaultCollate,
80    };
81    pub use super::transforms::{
82        CenterCrop, Compose, Normalize, RandomHorizontalFlip, Scale, Transform,
83    };
84}