Expand description
Data loading and preprocessing utilities for ToRSh
This crate provides PyTorch-compatible data loading functionality, including datasets, data loaders, and common transformations.
Re-exports§
pub use vision::Compose;pub use vision::ImageFolder;pub use vision::ImageNet;pub use vision::ImageToTensor;pub use vision::RandomHorizontalFlip;pub use vision::RandomRotation;pub use vision::RandomVerticalFlip;pub use vision::TensorToImage;pub use vision::TensorToVideo;pub use vision::VideoFolder;pub use vision::VideoFrames;pub use vision::VideoToTensor;pub use vision::CIFAR10;pub use vision::MNIST;pub use collate::collate_fn;pub use collate::AdaptiveBatchSampler;pub use collate::BucketBatchSampler;pub use collate::CachedCollate;pub use collate::Collate;pub use collate::DynamicBatchCollate;pub use collate::optimized_collate_fn;pub use collate::OptimizedCollate;pub use dataloader::simple_random_dataloader;pub use dataloader::DataLoader;pub use dataloader::DataLoaderBuilder;pub use dataloader::DataLoaderTrait;pub use dataset::random_split;pub use dataset::BufferedStreamingDataset;pub use dataset::CachedDataset;pub use dataset::ChainDataset;pub use dataset::ConcatDataset;pub use dataset::DataPipeline;pub use dataset::Dataset;pub use dataset::DatasetToStreaming;pub use dataset::InfiniteDataset;pub use dataset::IterableDataset;pub use dataset::PipelineStreamingDataset;pub use dataset::RealTimeDataset;pub use dataset::StreamingDataset;pub use dataset::Subset;pub use dataset::TensorDataset;pub use dataset::MemoryMappedDataset;pub use sampler::AcquisitionStrategy;pub use sampler::ActiveLearningSampler;pub use sampler::AdaptiveSampler;pub use sampler::AdaptiveStrategy;pub use sampler::BatchSampler;pub use sampler::BatchingSampler;pub use sampler::CurriculumSampler;pub use sampler::CurriculumStrategy;pub use sampler::DistributedSampler;pub use sampler::GroupedSampler;pub use sampler::ImportanceSampler;pub use sampler::RandomSampler;pub use sampler::Sampler;pub use sampler::SequentialSampler;pub use sampler::StratifiedSampler;pub use sampler::SubsetRandomSampler;pub use sampler::WeightedRandomSampler;pub use text::TextClassificationDataset;pub use text::TextFileDataset;pub use text::TextSequence;pub use text::TokenIdsToTensor;pub use text::Vocabulary;pub use error::diagnostics;pub use error::patterns;pub use error::recovery;pub use error::BatchInfo;pub use error::CollationErrorKind;pub use error::ConfigErrorKind;pub use error::DataError;pub use error::DataLoaderErrorKind;pub use error::DatasetErrorKind;pub use error::ErrorContext;pub use error::ErrorSeverity;pub use error::IoErrorKind;pub use error::ResourceErrorKind;pub use error::Result;pub use error::SamplerErrorKind;pub use error::TransformErrorKind;pub use error::WithContext;pub use transforms::Chain;pub use transforms::Compose as TransformCompose;pub use transforms::Conditional;pub use transforms::Lambda;pub use transforms::Normalize;pub use transforms::ToType;pub use transforms::Transform;pub use transforms::TransformBuilder;pub use transforms::TransformExt;pub use utils::batch;pub use utils::concurrent;pub use utils::create_size_tuple;pub use utils::errors;pub use utils::memory;pub use utils::performance;pub use utils::validate_dataset_path;pub use utils::validate_file_extension;pub use utils::validate_not_empty;pub use utils::validate_positive;pub use utils::validate_probability;pub use utils::validate_range;pub use utils::validate_same_length;pub use utils::validate_tensor_shape;pub use utils::Cacheable;pub use utils::Configurable;pub use utils::ProgressTracker;pub use utils::Resettable;pub use builtin::load_builtin_dataset;pub use builtin::make_blobs;pub use builtin::make_classification;pub use builtin::make_regression;pub use builtin::BuiltinDataset;pub use builtin::ClassificationConfig;pub use builtin::ClusteringConfig;pub use builtin::DatasetRegistry;pub use builtin::DatasetResult;pub use builtin::RegressionConfig;pub use builtin::ScalingMethod;pub use builtin::SyntheticDataConfig;pub use arrow_integration::arrow_utils;pub use arrow_integration::ArrowDataset;pub use tfrecord_integration::tfrecord_utils;pub use tfrecord_integration::Example;pub use tfrecord_integration::FeatureValue;pub use tfrecord_integration::TFRecordDataset;pub use tfrecord_integration::TFRecordDatasetBuilder;pub use tfrecord_integration::TFRecordError;pub use tfrecord_integration::TFRecordReader;pub use database_integration::database_utils;pub use database_integration::DatabaseBackend;pub use database_integration::DatabaseConfig;pub use database_integration::DatabaseConnection;pub use database_integration::DatabaseDataset;pub use database_integration::DatabaseDatasetBuilder;pub use database_integration::DatabaseError;pub use database_integration::DatabaseRow;pub use database_integration::DatabaseValue;
Modules§
- arrow_
integration - Apache Arrow integration for efficient data exchange
- augmentation_
pipeline - Data augmentation pipeline for machine learning training
- builtin
- Built-in datasets powered by SciRS2
- collate
- Batch collation functions
- database_
integration - Database integration for loading data from various database backends
- dataloader
- DataLoader implementation for efficient data loading
- dataset
- Dataset trait and implementations
- error
- Enhanced error handling for torsh-data
- online_
transforms - Online data augmentation system for real-time transform application
- prelude
- Prelude module for convenient imports
- sampler
- Unified Data Sampling Interface
- tensor_
transforms - Tensor transformation operations for computer vision and image processing
- text
- Text preprocessing and NLP dataset utilities
- text_
processing - Text processing and natural language processing transformations
- tfrecord_
integration - TensorFlow TFRecord format integration
- transforms
- Data transformation and augmentation framework for ToRSh
- utils
- Common utilities to reduce code duplication across modules
- vision
- Vision-specific datasets and transformations
Macros§
- builder_
pattern - Macro to create builder patterns with fluent interface
- simple_
random_ transform - Common transform implementation macro
- validated_
constructor - Macro to create a simple constructor with validation