Skip to main content

lerobot_dataset_io/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod arrow_columns;
4mod dataset;
5mod error;
6mod export;
7pub mod format_extension_support;
8mod info;
9#[cfg(feature = "object-store")]
10mod object_store_provider;
11mod path_template;
12mod source;
13mod time;
14mod vector_columns;
15
16use std::fmt::Debug;
17use std::path::Path;
18
19use async_trait::async_trait;
20
21pub use error::{ExportDestinationProblem, LerobotError};
22pub use export::{
23    CameraVideoWriteResult, DatasetExportExtension, EXPORT_STAGING_OWNERSHIP_MANIFEST_FILE_NAME,
24    ExportEpisodes, ExportRangePlacement, ExportReport, InvalidWrittenVideo, PreparedExportInputs,
25    SourceSegment, VideoSegmentWriter, WrittenVideo, export_episode_range_preserving_packed_videos,
26    export_subset, export_subset_preserving_packed_videos,
27    export_subset_preserving_packed_videos_with_extension, export_subset_with_extension,
28    write_packed_subset_info_json, write_packed_subset_info_json_with_extension,
29};
30pub use info::{DatasetInfo, DatasetInfoParseError, FeatureDtype, FeatureSpec, VideoFeatureKind};
31#[cfg(feature = "object-store")]
32pub use object_store_provider::{ObjectStoreDatasetFileProvider, ObjectStoreDatasetSourceError};
33pub use path_template::{
34    DatasetRelativePath, DatasetRelativePathError, PathTemplateRenderError, render_path_template,
35};
36pub use source::{DatasetCachePolicy, DatasetSource, RemoteDatasetSource};
37pub use time::{FramesPerSecond, InvalidFramesPerSecond, InvalidTimestampRange, TimestampRange};
38pub use vector_columns::{
39    EpisodeDataFileReadPlan, EpisodeDataReadPlan, EpisodeVectorColumn, EpisodeVectorColumns,
40    EpisodeVectorReadReport, EpisodeVectorReadSchema, EpisodeVectorRow, EpisodeVectorRowReader,
41    EpisodeVectorSelectionMode, InvalidEpisodeDataReadPlan, PlannedEpisodeDataRows,
42    read_episodes_vector_columns_from_file,
43};
44
45/// Opaque failure returned by a [`DatasetFileProvider`].
46///
47/// Dataset readers add operation, path, and source-identity context before
48/// exposing the failure. Callers never branch on provider-specific failures,
49/// so implementations can preserve their native error without forcing it into
50/// a shared enum.
51pub type DatasetFileProviderError = Box<dyn std::error::Error + Send + Sync + 'static>;
52
53/// Result of materializing one remote dataset file into a local cache.
54#[derive(Debug, Clone, Copy, PartialEq, Eq)]
55pub enum DatasetFileMaterialization {
56    /// The file existed and is atomically available at the requested path.
57    Materialized,
58    /// The provider has no file at the requested dataset-relative path.
59    NotFound,
60}
61
62/// The remote file capabilities required by a `LeRobot` dataset reader.
63///
64/// Implementations may wrap an object store, an HTTP service, or another
65/// backend. Returned file names must be relative to the dataset root and use
66/// forward slashes. Materialization must not leave a partial file at
67/// `destination_path` when it fails.
68#[async_trait]
69pub trait DatasetFileProvider: Debug + Send + Sync {
70    /// Lists every dataset-relative file beneath `relative_prefix`.
71    ///
72    /// `report_files_discovered` receives the cumulative number of files
73    /// observed while the backend listing advances, before the complete list
74    /// is available. The reader normalizes ordering after this boundary.
75    async fn list_files(
76        &self,
77        relative_prefix: &str,
78        report_files_discovered: &mut (dyn FnMut(u64) + Send),
79    ) -> Result<Vec<String>, DatasetFileProviderError>;
80
81    /// Atomically materializes `relative_path` into `destination_path`.
82    async fn materialize_file(
83        &self,
84        relative_path: &str,
85        destination_path: &Path,
86    ) -> Result<DatasetFileMaterialization, DatasetFileProviderError>;
87}
88
89pub use dataset::{
90    Dataset, DatasetOpenProgress, DatasetOpenProgressReporter, Episode, VideoSegment,
91};