use std::fs;
use std::path::Path;
use flodl::tensor::{Result, Tensor};
use flodl::{DType, Device};
use crate::utils::error::DataError;
#[derive(Clone, Debug)]
pub struct Dataset {
pub inputs: Tensor,
pub targets: Tensor,
}
impl Dataset {
pub fn len(&self) -> usize {
self.inputs.shape()[0] as usize
}
pub fn to_f32(&self) -> Result<Dataset> {
self.to_dtype(DType::Float32)
}
pub fn to_dtype(&self, dtype: DType) -> Result<Dataset> {
let cast = |t: &Tensor| -> Result<Tensor> {
if t.dtype() == dtype {
Ok(t.clone())
} else {
t.to_dtype(dtype)
}
};
Ok(Dataset {
inputs: cast(&self.inputs)?,
targets: cast(&self.targets)?,
})
}
pub fn to_device(&self, device: Device) -> Result<Dataset> {
let transfer = |t: &Tensor| -> Result<Tensor> {
if t.device() == device {
Ok(t.clone())
} else {
let data = t.to_f32_vec()?;
Tensor::from_f32(&data, &t.shape(), device)
}
};
Ok(Dataset {
inputs: transfer(&self.inputs)?,
targets: transfer(&self.targets)?,
})
}
}
const MAGIC: &[u8; 4] = b"GRA1";
const TAG_F32: u8 = 0;
const TAG_F64: u8 = 1;
const TAG_I64: u8 = 2;
fn dtype_tag(dtype: DType) -> std::result::Result<u8, DataError> {
match dtype {
DType::Float32 => Ok(TAG_F32),
DType::Float64 => Ok(TAG_F64),
DType::Int64 => Ok(TAG_I64),
other => Err(DataError::UnsupportedDtype(format!(
"{other:?} for tensor file (use Float32/Float64/Int64)"
))),
}
}
fn tag_dtype(tag: u8) -> std::result::Result<DType, DataError> {
match tag {
TAG_F32 => Ok(DType::Float32),
TAG_F64 => Ok(DType::Float64),
TAG_I64 => Ok(DType::Int64),
other => Err(DataError::UnknownDtypeTag(other)),
}
}
fn elem_size(dtype: DType) -> usize {
match dtype {
DType::Float32 => 4,
DType::Float64 => 8,
DType::Int64 => 8,
_ => 0,
}
}
pub fn save_tensor(path: &Path, t: &Tensor) -> Result<()> {
let dtype = t.dtype();
let tag = dtype_tag(dtype)?;
let shape = t.shape();
let blob = t.to_blob()?;
let mut out: Vec<u8> = Vec::with_capacity(8 + 8 + shape.len() * 8 + blob.len());
out.extend_from_slice(MAGIC);
out.push(tag);
out.extend_from_slice(&(shape.len() as u64).to_le_bytes());
for &d in &shape {
out.extend_from_slice(&(d as u64).to_le_bytes());
}
out.extend_from_slice(&blob);
fs::write(path, out).map_err(|source| DataError::Io {
path: path.display().to_string(),
source,
})?;
Ok(())
}
pub fn load_tensor(path: &Path) -> Result<Tensor> {
let path_str = path.display().to_string();
let bytes = fs::read(path).map_err(|source| DataError::Io {
path: path_str.clone(),
source,
})?;
if bytes.len() < 9 || &bytes[..4] != MAGIC {
return Err(DataError::BadMagic { path: path_str }.into());
}
let dtype = tag_dtype(bytes[4])?;
let mut off = 5usize;
let read_u64 = |off: &mut usize| -> std::result::Result<u64, DataError> {
if *off + 8 > bytes.len() {
return Err(DataError::Truncated("tensor header".to_string()));
}
let v = u64::from_le_bytes(bytes[*off..*off + 8].try_into().unwrap());
*off += 8;
Ok(v)
};
let ndim = read_u64(&mut off)? as usize;
let mut shape = Vec::with_capacity(ndim);
for _ in 0..ndim {
shape.push(read_u64(&mut off)? as i64);
}
let numel: i64 = shape.iter().product();
let expected = (numel as usize) * elem_size(dtype);
if off + expected != bytes.len() {
return Err(DataError::SizeMismatch {
path: path_str,
expected,
found: bytes.len() - off,
}
.into());
}
Tensor::from_blob(&bytes[off..], &shape, dtype, Device::CPU)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum DataFormat {
Bin,
Csv,
Both,
}
pub fn save_dataset_as(dir: &Path, ds: &Dataset, format: DataFormat) -> Result<()> {
match format {
DataFormat::Bin => save_dataset(dir, ds),
DataFormat::Csv => save_csv_dataset(dir, &ds.inputs, &ds.targets),
DataFormat::Both => {
save_dataset(dir, ds)?;
save_csv_dataset(dir, &ds.inputs, &ds.targets)
}
}
}
pub fn load_dataset_auto(dir: &Path) -> Result<Dataset> {
if dir.join("inputs.bin").exists() {
return load_dataset(dir);
}
if dir.join("inputs.csv").exists() {
return load_csv_dataset(dir);
}
Err(DataError::Io {
path: dir.display().to_string(),
source: std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("no inputs.bin or inputs.csv found in {}", dir.display()),
),
}
.into())
}
pub fn save_dataset(dir: &Path, ds: &Dataset) -> Result<()> {
fs::create_dir_all(dir).map_err(|source| DataError::Io {
path: dir.display().to_string(),
source,
})?;
save_tensor(&dir.join("inputs.bin"), &ds.inputs)?;
save_tensor(&dir.join("targets.bin"), &ds.targets)?;
let meta = serde_json::json!({
"inputs_shape": ds.inputs.shape(),
"targets_shape": ds.targets.shape(),
});
let meta = serde_json::to_string_pretty(&meta)
.map_err(|e| DataError::Json(format!("meta.json: {e}")))?;
fs::write(dir.join("meta.json"), meta).map_err(|source| DataError::Io {
path: dir.join("meta.json").display().to_string(),
source,
})?;
Ok(())
}
pub fn load_dataset(dir: &Path) -> Result<Dataset> {
let inputs = load_tensor(&dir.join("inputs.bin"))?;
let targets = load_tensor(&dir.join("targets.bin"))?;
Ok(Dataset { inputs, targets })
}
pub fn resolve_dataset(dir: &Path) -> Result<Dataset> {
let cache_dir = dir.join("flodl_data");
if cache_dir.join("inputs.bin").exists() {
return load_dataset(&cache_dir);
}
if dir.join("inputs.bin").exists() {
return load_dataset(dir);
}
if dir.join("inputs.csv").exists() {
let dataset = load_csv_dataset(dir)?;
std::fs::create_dir_all(&cache_dir)
.map_err(|e| flodl::tensor::TensorError::new(&e.to_string()))?;
save_dataset(&cache_dir, &dataset)?;
return Ok(dataset);
}
Err(DataError::Io {
path: dir.display().to_string(),
source: std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("no inputs.bin or inputs.csv found in {}", dir.display()),
),
}
.into())
}
pub fn load_csv_dataset(dir: &Path) -> Result<Dataset> {
let inputs_path = dir.join("inputs.csv");
let targets_path = dir.join("targets.csv");
if !inputs_path.exists() {
return Err(DataError::Io {
path: inputs_path.display().to_string(),
source: std::io::Error::new(std::io::ErrorKind::NotFound, "inputs.csv not found"),
}.into());
}
if !targets_path.exists() {
return Err(DataError::Io {
path: targets_path.display().to_string(),
source: std::io::Error::new(std::io::ErrorKind::NotFound, "targets.csv not found"),
}.into());
}
let inputs_raw = std::fs::read_to_string(&inputs_path).map_err(|e| DataError::Io {
path: inputs_path.display().to_string(),
source: e,
})?;
let targets_raw = std::fs::read_to_string(&targets_path).map_err(|e| DataError::Io {
path: targets_path.display().to_string(),
source: e,
})?;
let mut inputs_flat: Vec<f32> = Vec::new();
let mut input_dim: Option<usize> = None;
let mut n_samples: usize = 0;
let mut skipped_header = false;
for line in inputs_raw.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let row: Vec<f32> = match line.split(',')
.map(|s| s.trim().parse::<f32>())
.collect::<std::result::Result<Vec<_>, _>>()
{
Ok(row) => row,
Err(e) => {
if n_samples == 0 && !skipped_header {
skipped_header = true;
log::debug!("inputs.csv: skipping header row: {line}");
continue;
}
return Err(DataError::Csv(format!("inputs.csv parse error: {e}")).into());
}
};
if let Some(dim) = input_dim {
if row.len() != dim {
return Err(DataError::Csv(format!(
"inputs.csv: row {n_samples} has {} columns, expected {dim}", row.len()
)).into());
}
} else {
input_dim = Some(row.len());
}
inputs_flat.extend(row);
n_samples += 1;
}
let input_dim = input_dim.ok_or_else(|| DataError::Csv("inputs.csv is empty".into()))?;
let mut targets_flat: Vec<f32> = Vec::new();
let mut target_dim: Option<usize> = None;
let mut n_targets: usize = 0;
let mut skipped_header = false;
for line in targets_raw.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let row: Vec<f32> = match line.split(',')
.map(|s| s.trim().parse::<f32>())
.collect::<std::result::Result<Vec<_>, _>>()
{
Ok(row) => row,
Err(e) => {
if n_targets == 0 && !skipped_header {
skipped_header = true;
log::debug!("targets.csv: skipping header row: {line}");
continue;
}
return Err(DataError::Csv(format!("targets.csv parse error: {e}")).into());
}
};
if let Some(dim) = target_dim {
if row.len() != dim {
return Err(DataError::Csv(format!(
"targets.csv: row {n_targets} has {} columns, expected {dim}", row.len()
)).into());
}
} else {
target_dim = Some(row.len());
}
targets_flat.extend(row);
n_targets += 1;
}
let target_dim = target_dim.ok_or_else(|| DataError::Csv("targets.csv is empty".into()))?;
if n_samples != n_targets {
return Err(DataError::Csv(format!(
"sample count mismatch: inputs has {n_samples}, targets has {n_targets}"
)).into());
}
let inputs = Tensor::from_f32(&inputs_flat, &[n_samples as i64, input_dim as i64], Device::CPU)?;
let targets = Tensor::from_f32(&targets_flat, &[n_targets as i64, target_dim as i64], Device::CPU)?;
Ok(Dataset { inputs, targets })
}
pub fn save_csv_dataset(dir: &Path, inputs: &Tensor, targets: &Tensor) -> Result<()> {
fs::create_dir_all(dir).map_err(|source| DataError::Io {
path: dir.display().to_string(),
source,
})?;
let in_data = inputs.to_f32_vec().map_err(|e| DataError::Csv(format!("inputs to_f32_vec: {e}")))?;
let tgt_data = targets.to_f32_vec().map_err(|e| DataError::Csv(format!("targets to_f32_vec: {e}")))?;
let in_shape = inputs.shape();
let tgt_shape = targets.shape();
let (n_in, dim_in) = (in_shape[0] as usize, in_shape[1] as usize);
let (n_tgt, dim_tgt) = (tgt_shape[0] as usize, tgt_shape[1] as usize);
let mut inputs_csv = String::with_capacity(n_in * dim_in * 8);
for row in 0..n_in {
let start = row * dim_in;
let line: Vec<String> = (0..dim_in)
.map(|c| format!("{:.6}", in_data[start + c]))
.collect();
inputs_csv.push_str(&line.join(","));
inputs_csv.push('\n');
}
fs::write(dir.join("inputs.csv"), inputs_csv).map_err(|source| DataError::Io {
path: dir.join("inputs.csv").display().to_string(),
source,
})?;
let mut targets_csv = String::with_capacity(n_tgt * dim_tgt * 8);
for row in 0..n_tgt {
let start = row * dim_tgt;
let line: Vec<String> = (0..dim_tgt)
.map(|c| format!("{:.6}", tgt_data[start + c]))
.collect();
targets_csv.push_str(&line.join(","));
targets_csv.push('\n');
}
fs::write(dir.join("targets.csv"), targets_csv).map_err(|source| DataError::Io {
path: dir.join("targets.csv").display().to_string(),
source,
})?;
Ok(())
}
pub fn synthetic_classification(
n: usize,
in_dim: usize,
out_dim: usize,
seed: u64,
device: Device,
) -> Result<Dataset> {
let mut rng = fastrand::Rng::with_seed(seed);
let inputs: Vec<f32> = (0..n * in_dim).map(|_| rng.f32()).collect();
let inputs = Tensor::from_f32(&inputs, &[n as i64, in_dim as i64], device)?;
let mut targets = vec![0.0f32; n * out_dim];
for row in 0..n {
let c = rng.usize(0..out_dim);
targets[row * out_dim + c] = 1.0;
}
let targets = Tensor::from_f32(&targets, &[n as i64, out_dim as i64], device)?;
Ok(Dataset { inputs, targets })
}
pub fn make_sine(n: usize) -> (Tensor, Tensor) {
let mut rng = fastrand::Rng::with_seed(42);
let inputs: Vec<f32> = (0..n).map(|_| rng.f32() * 2.0 - 1.0).collect();
let targets: Vec<f32> = inputs.iter().map(|x| (x * std::f32::consts::TAU).sin()).collect();
let inputs = Tensor::from_f32(&inputs, &[n as i64, 1], Device::CPU).unwrap();
let targets = Tensor::from_f32(&targets, &[n as i64, 1], Device::CPU).unwrap();
(inputs, targets)
}
pub fn make_xor(n: usize) -> (Tensor, Tensor) {
let mut rng = fastrand::Rng::with_seed(42);
let mut inputs = Vec::with_capacity(n * 2);
let mut targets = Vec::with_capacity(n * 2);
for _ in 0..n {
let a = rng.f32();
let b = rng.f32();
inputs.push(a);
inputs.push(b);
let xor = if (a > 0.5) ^ (b > 0.5) { 1.0 } else { 0.0 };
targets.push(xor);
targets.push(1.0 - xor);
}
let inputs = Tensor::from_f32(&inputs, &[n as i64, 2], Device::CPU).unwrap();
let targets = Tensor::from_f32(&targets, &[n as i64, 2], Device::CPU).unwrap();
(inputs, targets)
}
pub fn split_indices(
len: usize,
train_ratio: f32,
eval_ratio: f32,
seed: u64,
) -> (Vec<i64>, Vec<i64>) {
assert!((train_ratio + eval_ratio - 1.0).abs() < 1e-6, "ratios must sum to 1.0");
let mut indices: Vec<i64> = (0..len as i64).collect();
let mut rng = fastrand::Rng::with_seed(seed);
for i in (1..indices.len()).rev() {
let j = rng.usize(0..=i);
indices.swap(i, j);
}
let split = (len as f32 * train_ratio).round() as usize;
let train: Vec<i64> = indices[..split].to_vec();
let eval: Vec<i64> = indices[split..].to_vec();
if eval.is_empty() {
static WARNED: std::sync::Once = std::sync::Once::new();
WARNED.call_once(|| {
log::warn!(
"eval set is empty (train_ratio={train_ratio}) — falling back to using ALL data for eval (train/eval overlap)"
);
});
(train, indices.clone())
} else {
(train, eval)
}
}
#[cfg(test)]
mod tests {
use super::*;
use proptest::prelude::*;
use std::path::PathBuf;
fn temp_dir() -> PathBuf {
std::env::temp_dir().join(format!("gras_data_test_{}", fastrand::u64(..)))
}
#[test]
fn test_tensor_roundtrip() {
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
let t = Tensor::from_f32(&[1.0, 2.0, 3.0, 4.0], &[2, 2], Device::CPU).unwrap();
let path = dir.join("t.bin");
save_tensor(&path, &t).unwrap();
let loaded = load_tensor(&path).unwrap();
assert_eq!(loaded.shape(), t.shape());
assert_eq!(loaded.to_f32_vec().unwrap(), t.to_f32_vec().unwrap());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_dataset_roundtrip() {
let dir = temp_dir();
let inputs = Tensor::from_f32(&[1.0, 2.0, 3.0, 4.0], &[2, 2], Device::CPU).unwrap();
let targets = Tensor::from_f32(&[5.0, 6.0, 7.0, 8.0], &[2, 2], Device::CPU).unwrap();
let ds = Dataset { inputs, targets };
save_dataset(&dir, &ds).unwrap();
let loaded = load_dataset(&dir).unwrap();
assert_eq!(loaded.inputs.shape(), ds.inputs.shape());
assert_eq!(loaded.targets.shape(), ds.targets.shape());
assert_eq!(
loaded.inputs.to_f32_vec().unwrap(),
ds.inputs.to_f32_vec().unwrap()
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_load_tensor_rejects_garbage() {
let dir = temp_dir();
let path = dir.join("bad.bin");
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(&path, b"not a gras tensor").unwrap();
assert!(load_tensor(&path).is_err());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_csv_roundtrip() {
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("inputs.csv"), "1.0,2.0\n3.0,4.0\n5.0,6.0\n").unwrap();
std::fs::write(dir.join("targets.csv"), "0.0\n1.0\n0.0\n").unwrap();
let ds = load_csv_dataset(&dir).unwrap();
assert_eq!(ds.inputs.shape(), &[3, 2]);
assert_eq!(ds.targets.shape(), &[3, 1]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_csv_with_header() {
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("inputs.csv"), "feat1,feat2\n1.0,2.0\n3.0,4.0\n").unwrap();
std::fs::write(dir.join("targets.csv"), "label\n0.0\n1.0\n").unwrap();
let ds = load_csv_dataset(&dir).unwrap();
assert_eq!(ds.inputs.shape(), &[2, 2]);
assert_eq!(ds.targets.shape(), &[2, 1]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_csv_sample_count_mismatch() {
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("inputs.csv"), "1.0,2.0\n3.0,4.0\n5.0,6.0\n").unwrap();
std::fs::write(dir.join("targets.csv"), "0.0\n1.0\n").unwrap(); let result = load_csv_dataset(&dir);
assert!(result.is_err());
let err = result.unwrap_err().to_string();
assert!(err.contains("sample count mismatch"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_csv_skips_comments() {
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("inputs.csv"), "# header comment\n1.0,2.0\n3.0,4.0\n").unwrap();
std::fs::write(dir.join("targets.csv"), "# another comment\n0.0\n1.0\n").unwrap();
let ds = load_csv_dataset(&dir).unwrap();
assert_eq!(ds.inputs.shape(), &[2, 2]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_save_dataset_as_roundtrip_all_formats() {
let mut rng = fastrand::Rng::with_seed(7);
let data: Vec<f32> = (0..24).map(|_| rng.f32()).collect();
let ds = Dataset {
inputs: Tensor::from_f32(&data, &[8, 3], Device::CPU).unwrap(),
targets: Tensor::from_f32(&[0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0], &[8, 1], Device::CPU).unwrap(),
};
for fmt in [DataFormat::Bin, DataFormat::Csv, DataFormat::Both] {
let dir = std::env::temp_dir().join(format!("gras_data_fmt_{:?}", fmt));
let _ = std::fs::remove_dir_all(&dir);
save_dataset_as(&dir, &ds, fmt).unwrap();
let loaded = load_dataset_auto(&dir).unwrap();
assert_eq!(loaded.inputs.shape(), ds.inputs.shape());
assert_eq!(loaded.targets.shape(), ds.targets.shape());
let _ = std::fs::remove_dir_all(&dir);
}
let dir = std::env::temp_dir().join("gras_data_fmt_Both_files");
let _ = std::fs::remove_dir_all(&dir);
save_dataset_as(&dir, &ds, DataFormat::Both).unwrap();
assert!(dir.join("inputs.bin").exists(), "Bin part of Both");
assert!(dir.join("inputs.csv").exists(), "Csv part of Both");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_load_dataset_auto_prefers_bin() {
let mut rng = fastrand::Rng::with_seed(11);
let data: Vec<f32> = (0..16).map(|_| rng.f32()).collect();
let ds = Dataset {
inputs: Tensor::from_f32(&data, &[8, 2], Device::CPU).unwrap(),
targets: Tensor::from_f32(&data, &[8, 2], Device::CPU).unwrap(),
};
let dir = std::env::temp_dir().join("gras_data_auto_prefer");
let _ = std::fs::remove_dir_all(&dir);
save_dataset_as(&dir, &ds, DataFormat::Both).unwrap();
let loaded = load_dataset_auto(&dir).unwrap();
assert_eq!(loaded.inputs.to_f32_vec().unwrap(), data, "must load .bin");
let _ = std::fs::remove_dir_all(&dir);
}
proptest! {
#[test]
fn prop_tensor_roundtrip(
rows in 1usize..8,
cols in 1usize..8,
seed in 0u64..1_000_000,
) {
let mut rng = fastrand::Rng::with_seed(seed);
let data: Vec<f32> = (0..rows * cols).map(|_| rng.f32()).collect();
let t = Tensor::from_f32(&data, &[rows as i64, cols as i64], Device::CPU).unwrap();
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("t.bin");
save_tensor(&path, &t).unwrap();
let loaded = load_tensor(&path).unwrap();
prop_assert_eq!(loaded.shape(), t.shape());
prop_assert_eq!(loaded.dtype(), t.dtype());
prop_assert_eq!(loaded.to_f32_vec().unwrap(), t.to_f32_vec().unwrap());
let _ = std::fs::remove_dir_all(&dir);
}
}
}
pub fn one_hot(indices: &[usize], num_classes: usize, device: Device) -> Result<Tensor> {
let n = indices.len();
let mut data = vec![0.0f32; n * num_classes];
for (i, &c) in indices.iter().enumerate() {
data[i * num_classes + c] = 1.0;
}
Tensor::from_f32(&data, &[n as i64, num_classes as i64], device)
}