alimentar 0.2.7

Data Loading, Distribution and Tooling in Pure Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

//! alimentar - Data Loading, Distribution and Tooling in Pure Rust
//!
//! A sovereign-first data loading library for the paiml AI stack.
//! Provides HuggingFace-compatible functionality without mandatory cloud
//! dependency.
//!
//! # Design Principles
//!
//! 1. **Sovereign-first** - Local storage default, no mandatory cloud
//!    dependency
//! 2. **Pure Rust** - No Python, no FFI (WASM-compatible)
//! 3. **Zero-copy** - Arrow `RecordBatch` throughout
//! 4. **Ecosystem aligned** - Arrow 53, Parquet 53
//!
//! # Quick Start
//!
//! ```no_run
//! use alimentar::{ArrowDataset, DataLoader};
//!
//! // Load a parquet file
//! let dataset = ArrowDataset::from_parquet("data/train.parquet").unwrap();
//!
//! // Create a data loader
//! let loader = DataLoader::new(dataset).batch_size(32).shuffle(true);
//!
//! // Iterate over batches
//! for batch in loader {
//!     println!("Batch with {} rows", batch.num_rows());
//! }
//! ```

// unsafe_code is forbidden except where explicitly allowed (e.g., mmap module)
#![deny(unsafe_code)]
#![deny(missing_docs)]
#![deny(clippy::unwrap_used)]
#![deny(clippy::expect_used)]
// Allow common test patterns
#![cfg_attr(
    test,
    allow(
        clippy::unwrap_used,
        clippy::expect_used,
        clippy::cast_lossless,
        clippy::cast_possible_truncation,
        clippy::cast_possible_wrap,
        clippy::cast_precision_loss,
        clippy::cloned_ref_to_slice_refs,
        clippy::redundant_clone,
        clippy::needless_collect,
        clippy::too_many_lines,
        clippy::for_kv_map,
        clippy::bool_to_int_with_if,
        clippy::float_cmp,
        clippy::iter_on_single_items,
        clippy::similar_names,
        clippy::unreadable_literal
    )
)]
// Allow some pedantic lints for cleaner code
#![allow(clippy::doc_markdown)]
#![allow(clippy::uninlined_format_args)]
#![allow(clippy::redundant_closure_for_method_calls)]
#![allow(clippy::map_unwrap_or)]
#![allow(clippy::useless_conversion)]

#[cfg(feature = "tokio-runtime")]
pub mod async_prefetch;
pub mod backend;
/// CLI module for command-line interface
#[cfg(feature = "cli")]
pub mod cli;
pub mod dataloader;
pub mod dataset;
pub mod datasets;
pub mod drift;
pub mod error;
pub mod federated;
pub mod format;
#[cfg(feature = "hf-hub")]
pub mod hf_hub;
pub mod imbalance;
#[cfg(feature = "mmap")]
pub mod mmap;
pub mod parallel;
pub mod quality;
pub mod registry;
pub mod serve;
pub mod sketch;
pub mod split;
pub mod streaming;
pub mod tensor;
pub mod transform;
#[cfg(feature = "shuffle")]
pub mod weighted;

#[cfg(feature = "doctest")]
pub mod doctest;
#[cfg(feature = "repl")]
pub mod repl;
/// TUI dataset viewer module
pub mod tui;

// Re-exports for convenience
// Re-export arrow types commonly needed
pub use arrow::{
    array::RecordBatch,
    datatypes::{Schema, SchemaRef},
};
#[cfg(feature = "tokio-runtime")]
pub use async_prefetch::{AsyncPrefetchBuilder, AsyncPrefetchDataset, SyncPrefetchDataset};
pub use dataloader::DataLoader;
pub use dataset::{ArrowDataset, CsvOptions, Dataset, JsonOptions};
#[cfg(feature = "doctest")]
pub use doctest::{DocTest, DocTestCorpus, DocTestParser};
pub use drift::{ColumnDrift, DriftDetector, DriftReport, DriftSeverity, DriftTest};
pub use error::{Error, Result};
pub use federated::{
    FederatedSplitCoordinator, FederatedSplitStrategy, GlobalSplitReport, NodeSplitInstruction,
    NodeSplitManifest, NodeSummary, SplitQualityIssue,
};
pub use imbalance::{
    ClassDistribution, ImbalanceDetector, ImbalanceMetrics, ImbalanceRecommendation,
    ImbalanceReport, ImbalanceSeverity, ResampleStrategy, sqrt_inverse_weights,
};
#[cfg(feature = "shuffle")]
pub use imbalance::resample;
#[cfg(feature = "mmap")]
pub use mmap::{MmapDataset, MmapDatasetBuilder};
pub use parallel::{ParallelDataLoader, ParallelDataLoaderBuilder};
pub use quality::{ColumnQuality, QualityChecker, QualityIssue, QualityProfile, QualityReport, TextColumnStats};
pub use sketch::{
    Centroid, DDSketch, DataSketch, DistributedDriftDetector, SketchDriftResult, SketchType,
    TDigest,
};
pub use split::DatasetSplit;
pub use transform::{
    Cast, Chain, Drop, FillNull, FillStrategy, Filter, Map, NormMethod, Normalize, Rename, Select,
    Skip, Sort, SortOrder, Take, Transform, Unique,
};
#[cfg(feature = "shuffle")]
pub use transform::{Fim, FimFormat, FimTokens, Sample, Shuffle};
pub use tui::{DatasetAdapter, DatasetViewer, RowDetailView, SchemaInspector, TuiError, TuiResult};
#[cfg(feature = "shuffle")]
pub use weighted::WeightedDataLoader;