rskit_dataset/
lib.rs

1//! Dataset collection framework — streaming sources, transforms, and schema validation on a generic collection engine.
2//!
3//! One generic [`Collector<T>`] drives collection for every item family: it is generic over any [`DatasetItem`], so [`DataItem`] blobs and [`DatasetRecord`] rows share the same worker pool, cancellation, and event loop. The engine stays item-agnostic — it never writes items itself. Per-item materialization lives behind an injected [`ItemSink<T>`] ([`LocalBlobSink`] writes [`DataItem`] samples to `real/` and `ai/`), and per-item validation is a pluggable [`Validator<T>`] that callers opt into (for example a schema-backed validator for tabular records).
4
5#![warn(missing_docs)]
6
7pub mod collector;
8mod item;
9mod label;
10mod limits;
11pub mod manifest;
12mod media;
13mod payload;
14pub mod record;
15pub mod schema;
16mod sink;
17pub mod source;
18pub mod stream;
19pub mod target;
20pub mod transform;
21mod validate;
22
23pub use collector::{Collector, CollectorConfig, CollectorResult, NullProgress, ProgressCallback};
24pub use item::{DataItem, DatasetItem};
25pub use label::Label;
26pub use limits::{DEFAULT_MAX_IN_MEMORY_BYTES, DatasetLimits};
27pub use manifest::{CacheStatus, Manifest, SourceEntry, SourceStats};
28pub use media::MediaType;
29pub use payload::DataPayload;
30pub use record::{
31    BoxRecordStream, CsvReader, CsvWriter, DatasetFormat, DatasetReader, DatasetRecord,
32    DatasetWriter, JsonArrayReader, JsonArrayWriter, JsonLinesReader, JsonLinesWriter, RecordSink,
33    RecordSource, SchemaValidator, filter_records, select_columns,
34};
35pub use schema::{DatasetSchema, validate_record};
36pub use sink::{ItemSink, LocalBlobSink};
37pub use source::{BoxDataStream, BoxItemStream, Source};
38pub use stream::DatasetStreamExt;
39pub use target::{PublishResult, Target};
40#[cfg(feature = "image-transform")]
41pub use transform::ResizeTransform;
42pub use transform::Transform;
43pub use validate::Validator;
44
45#[cfg(test)]
46mod tests;
rskit_dataset/lib.rs

rskit_dataset/
lib.rs
