lerobot_dataset_io/
lib.rs1#![doc = include_str!("../README.md")]
2
3mod arrow_columns;
4mod dataset;
5mod error;
6mod export;
7pub mod format_extension_support;
8mod info;
9#[cfg(feature = "object-store")]
10mod object_store_provider;
11mod path_template;
12mod source;
13mod time;
14mod vector_columns;
15
16use std::fmt::Debug;
17use std::path::Path;
18
19use async_trait::async_trait;
20
21pub use error::{ExportDestinationProblem, LerobotError};
22pub use export::{
23 CameraVideoWriteResult, DatasetExportExtension, EXPORT_STAGING_OWNERSHIP_MANIFEST_FILE_NAME,
24 ExportEpisodes, ExportRangePlacement, ExportReport, InvalidWrittenVideo, PreparedExportInputs,
25 SourceSegment, VideoSegmentWriter, WrittenVideo, export_episode_range_preserving_packed_videos,
26 export_subset, export_subset_preserving_packed_videos,
27 export_subset_preserving_packed_videos_with_extension, export_subset_with_extension,
28 write_packed_subset_info_json, write_packed_subset_info_json_with_extension,
29};
30pub use info::{DatasetInfo, DatasetInfoParseError, FeatureDtype, FeatureSpec, VideoFeatureKind};
31#[cfg(feature = "object-store")]
32pub use object_store_provider::{ObjectStoreDatasetFileProvider, ObjectStoreDatasetSourceError};
33pub use path_template::{
34 DatasetRelativePath, DatasetRelativePathError, PathTemplateRenderError, render_path_template,
35};
36pub use source::{DatasetCachePolicy, DatasetSource, RemoteDatasetSource};
37pub use time::{FramesPerSecond, InvalidFramesPerSecond, InvalidTimestampRange, TimestampRange};
38pub use vector_columns::{
39 EpisodeDataFileReadPlan, EpisodeDataReadPlan, EpisodeVectorColumn, EpisodeVectorColumns,
40 EpisodeVectorReadReport, EpisodeVectorReadSchema, EpisodeVectorRow, EpisodeVectorRowReader,
41 EpisodeVectorSelectionMode, InvalidEpisodeDataReadPlan, PlannedEpisodeDataRows,
42 read_episodes_vector_columns_from_file,
43};
44
45pub type DatasetFileProviderError = Box<dyn std::error::Error + Send + Sync + 'static>;
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
55pub enum DatasetFileMaterialization {
56 Materialized,
58 NotFound,
60}
61
62#[async_trait]
69pub trait DatasetFileProvider: Debug + Send + Sync {
70 async fn list_files(
76 &self,
77 relative_prefix: &str,
78 report_files_discovered: &mut (dyn FnMut(u64) + Send),
79 ) -> Result<Vec<String>, DatasetFileProviderError>;
80
81 async fn materialize_file(
83 &self,
84 relative_path: &str,
85 destination_path: &Path,
86 ) -> Result<DatasetFileMaterialization, DatasetFileProviderError>;
87}
88
89pub use dataset::{
90 Dataset, DatasetOpenProgress, DatasetOpenProgressReporter, Episode, VideoSegment,
91};