entrenar 0.7.11

Training & Optimization library with autograd, LoRA, quantization, and model merging
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

//! Main batch loading entry point

use super::super::demo::create_demo_batches;
use crate::config::schema::TrainSpec;
use crate::error::Result;
use crate::train::Batch;

#[cfg(not(target_arch = "wasm32"))]
use super::json::load_json_batches;
#[cfg(all(not(target_arch = "wasm32"), feature = "parquet"))]
use super::parquet::load_parquet_batches;

/// Load training batches from data file using alimentar
///
/// Supports parquet, JSON, and CSV formats via alimentar.
/// Falls back to demo data if the file doesn't exist (for testing).
pub fn load_training_batches(spec: &TrainSpec) -> Result<Vec<Batch>> {
    let data_path = &spec.data.train;
    let batch_size = spec.data.batch_size;

    // Check if data file exists
    if !data_path.exists() {
        eprintln!("Warning: Training data not found at '{}', using demo data", data_path.display());
        return Ok(create_demo_batches(batch_size));
    }

    // Load data using alimentar (only on non-WASM)
    #[cfg(not(target_arch = "wasm32"))]
    {
        let ext = data_path.extension().and_then(|e| e.to_str()).unwrap_or("").to_lowercase();

        match ext.as_str() {
            #[cfg(feature = "parquet")]
            "parquet" => load_parquet_batches(data_path, batch_size),
            #[cfg(not(feature = "parquet"))]
            "parquet" => {
                eprintln!(
                    "Warning: Parquet support requires the 'parquet' feature, using demo data"
                );
                Ok(create_demo_batches(batch_size))
            }
            "json" => load_json_batches(data_path, batch_size),
            _ => {
                eprintln!("Warning: Unsupported data format '{ext}', using demo data");
                Ok(create_demo_batches(batch_size))
            }
        }
    }

    #[cfg(target_arch = "wasm32")]
    {
        eprintln!("Warning: Data loading not available in WASM, using demo data");
        Ok(create_demo_batches(batch_size))
    }
}