alimentar/
lib.rs

1//! alimentar - Data Loading, Distribution and Tooling in Pure Rust
2//!
3//! A sovereign-first data loading library for the paiml AI stack.
4//! Provides HuggingFace-compatible functionality without mandatory cloud
5//! dependency.
6//!
7//! # Design Principles
8//!
9//! 1. **Sovereign-first** - Local storage default, no mandatory cloud
10//!    dependency
11//! 2. **Pure Rust** - No Python, no FFI (WASM-compatible)
12//! 3. **Zero-copy** - Arrow `RecordBatch` throughout
13//! 4. **Ecosystem aligned** - Arrow 53, Parquet 53
14//!
15//! # Quick Start
16//!
17//! ```no_run
18//! use alimentar::{ArrowDataset, DataLoader};
19//!
20//! // Load a parquet file
21//! let dataset = ArrowDataset::from_parquet("data/train.parquet").unwrap();
22//!
23//! // Create a data loader
24//! let loader = DataLoader::new(dataset).batch_size(32).shuffle(true);
25//!
26//! // Iterate over batches
27//! for batch in loader {
28//!     println!("Batch with {} rows", batch.num_rows());
29//! }
30//! ```
31// unsafe_code is forbidden except where explicitly allowed (e.g., mmap module)
32#![deny(unsafe_code)]
33// APR-MONO: missing_docs relaxed to the workspace policy (`missing_docs = "allow"`, doc
34// coverage checked separately) now that this ex-repo crate is linted in-tree.
35#![allow(missing_docs)]
36#![allow(clippy::trivially_copy_pass_by_ref)] // APR-MONO: workspace policy (API consistency)
37#![allow(clippy::unnecessary_wraps)] // APR-MONO: workspace policy (Result wrappers for API consistency)
38#![allow(clippy::unwrap_used)]
39#![allow(clippy::expect_used)]
40#![allow(clippy::cast_precision_loss)]
41#![allow(clippy::cast_sign_loss)]
42#![allow(clippy::cast_possible_truncation)]
43#![allow(clippy::cast_possible_wrap)]
44#![allow(clippy::cast_lossless)]
45#![allow(clippy::approx_constant)]
46#![allow(clippy::len_zero)]
47#![allow(clippy::redundant_closure)]
48#![allow(clippy::redundant_clone)]
49#![allow(clippy::float_cmp)]
50#![allow(clippy::unreadable_literal)]
51#![allow(clippy::needless_collect)]
52#![allow(clippy::too_many_lines)]
53#![allow(clippy::bool_to_int_with_if)]
54#![allow(clippy::similar_names)]
55#![allow(clippy::doc_markdown)]
56#![allow(clippy::uninlined_format_args)]
57#![allow(clippy::redundant_closure_for_method_calls)]
58#![allow(clippy::map_unwrap_or)]
59#![allow(clippy::useless_conversion)]
60#![allow(clippy::iter_on_single_items)]
61#![allow(clippy::suboptimal_flops)]
62#![allow(clippy::cloned_ref_to_slice_refs)]
63#[macro_use]
64#[allow(unused_macros)]
65mod generated_contracts;
66#[cfg(feature = "tokio-runtime")]
67pub mod async_prefetch;
68pub mod backend;
69/// CLI module for command-line interface
70#[cfg(feature = "cli")]
71pub mod cli;
72pub mod dataloader;
73pub mod dataset;
74pub mod datasets;
75#[cfg(feature = "doctest")]
76pub mod doctest;
77pub mod drift;
78pub mod error;
79pub mod federated;
80pub mod format;
81#[cfg(feature = "hf-hub")]
82pub mod hf_hub;
83pub mod imbalance;
84#[cfg(feature = "mmap")]
85pub mod mmap;
86pub mod parallel;
87pub mod quality;
88pub mod registry;
89#[cfg(feature = "repl")]
90pub mod repl;
91pub mod serve;
92pub mod sketch;
93pub mod split;
94pub mod streaming;
95pub mod tensor;
96pub mod transform;
97/// TUI dataset viewer module
98pub mod tui;
99#[cfg(feature = "shuffle")]
100pub mod weighted;
101// Re-exports for convenience
102// Re-export arrow types commonly needed
103pub use arrow::{
104    array::RecordBatch,
105    datatypes::{Schema, SchemaRef},
106};
107#[cfg(feature = "tokio-runtime")]
108pub use async_prefetch::{AsyncPrefetchBuilder, AsyncPrefetchDataset, SyncPrefetchDataset};
109pub use dataloader::DataLoader;
110pub use dataset::{ArrowDataset, CsvOptions, Dataset, JsonOptions};
111#[cfg(feature = "doctest")]
112pub use doctest::{DocTest, DocTestCorpus, DocTestParser};
113pub use drift::{ColumnDrift, DriftDetector, DriftReport, DriftSeverity, DriftTest};
114pub use error::{Error, Result};
115pub use federated::{
116    FederatedSplitCoordinator, FederatedSplitStrategy, GlobalSplitReport, NodeSplitInstruction,
117    NodeSplitManifest, NodeSummary, SplitQualityIssue,
118};
119#[cfg(feature = "shuffle")]
120pub use imbalance::resample;
121pub use imbalance::{
122    sqrt_inverse_weights, ClassDistribution, ImbalanceDetector, ImbalanceMetrics,
123    ImbalanceRecommendation, ImbalanceReport, ImbalanceSeverity, ResampleStrategy,
124};
125#[cfg(feature = "mmap")]
126pub use mmap::{MmapDataset, MmapDatasetBuilder};
127pub use parallel::{ParallelDataLoader, ParallelDataLoaderBuilder};
128pub use quality::{
129    ColumnQuality, QualityChecker, QualityIssue, QualityProfile, QualityReport, TextColumnStats,
130};
131pub use sketch::{
132    Centroid, DDSketch, DataSketch, DistributedDriftDetector, SketchDriftResult, SketchType,
133    TDigest,
134};
135pub use split::DatasetSplit;
136pub use transform::{
137    Cast, Chain, Drop, FillNull, FillStrategy, Filter, Map, NormMethod, Normalize, Rename, Select,
138    Skip, Sort, SortOrder, Take, Transform, Unique,
139};
140#[cfg(feature = "shuffle")]
141pub use transform::{Fim, FimFormat, FimTokens, Sample, Shuffle};
142pub use tui::{DatasetAdapter, DatasetViewer, RowDetailView, SchemaInspector, TuiError, TuiResult};
143#[cfg(feature = "shuffle")]
144pub use weighted::WeightedDataLoader;
alimentar/lib.rs

alimentar/
lib.rs
