use std::fs;
use std::path::Path;
use flodl::tensor::{Result, Tensor};
use flodl::{DType, Device};
use crate::utils::error::DataError;
#[derive(Clone, Debug)]
pub struct Dataset {
pub inputs: Tensor,
pub targets: Tensor,
}
impl Dataset {
pub fn len(&self) -> usize {
self.inputs.shape()[0] as usize
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn dims(&self) -> (usize, usize) {
(
self.inputs.shape()[1] as usize,
self.targets.shape()[1] as usize,
)
}
pub fn from_tensors(inputs: Tensor, targets: Tensor) -> Result<Dataset> {
let rows = |t: &Tensor| t.shape().first().copied().unwrap_or(0) as usize;
let (rows_in, rows_out) = (rows(&inputs), rows(&targets));
if rows_in != rows_out {
return Err(DataError::RowCountMismatch {
inputs: rows_in,
targets: rows_out,
}
.into());
}
Ok(Dataset { inputs, targets })
}
pub fn to_f32(&self) -> Result<Dataset> {
self.to_dtype(DType::Float32)
}
pub fn to_dtype(&self, dtype: DType) -> Result<Dataset> {
let cast = |t: &Tensor| -> Result<Tensor> {
if t.dtype() == dtype {
Ok(t.clone())
} else {
t.to_dtype(dtype)
}
};
Ok(Dataset {
inputs: cast(&self.inputs)?,
targets: cast(&self.targets)?,
})
}
pub fn to_device(&self, device: Device) -> Result<Dataset> {
let transfer = |t: &Tensor| -> Result<Tensor> {
if t.device() == device {
Ok(t.clone())
} else {
let data = t.to_f32_vec()?;
Tensor::from_f32(&data, &t.shape(), device)
}
};
Ok(Dataset {
inputs: transfer(&self.inputs)?,
targets: transfer(&self.targets)?,
})
}
pub fn concat(&self, other: &Dataset) -> Result<Dataset> {
Ok(Dataset {
inputs: Tensor::cat_many(&[&self.inputs, &other.inputs], 0)?,
targets: Tensor::cat_many(&[&self.targets, &other.targets], 0)?,
})
}
}
const MAGIC: &[u8; 4] = b"GRA1";
const TAG_F32: u8 = 0;
const TAG_F64: u8 = 1;
const TAG_I64: u8 = 2;
fn dtype_tag(dtype: DType) -> std::result::Result<u8, DataError> {
match dtype {
DType::Float32 => Ok(TAG_F32),
DType::Float64 => Ok(TAG_F64),
DType::Int64 => Ok(TAG_I64),
other => Err(DataError::UnsupportedDtype(format!(
"{other:?} for tensor file (use Float32/Float64/Int64)"
))),
}
}
fn tag_dtype(tag: u8) -> std::result::Result<DType, DataError> {
match tag {
TAG_F32 => Ok(DType::Float32),
TAG_F64 => Ok(DType::Float64),
TAG_I64 => Ok(DType::Int64),
other => Err(DataError::UnknownDtypeTag(other)),
}
}
fn elem_size(dtype: DType) -> usize {
match dtype {
DType::Float32 => 4,
DType::Float64 => 8,
DType::Int64 => 8,
_ => 0,
}
}
pub fn save_tensor(path: &Path, t: &Tensor) -> Result<()> {
let dtype = t.dtype();
let tag = dtype_tag(dtype)?;
let shape = t.shape();
let blob = t.to_blob()?;
let mut out: Vec<u8> = Vec::with_capacity(8 + 8 + shape.len() * 8 + blob.len());
out.extend_from_slice(MAGIC);
out.push(tag);
out.extend_from_slice(&(shape.len() as u64).to_le_bytes());
for &d in &shape {
out.extend_from_slice(&(d as u64).to_le_bytes());
}
out.extend_from_slice(&blob);
fs::write(path, out).map_err(|source| DataError::Io {
path: path.display().to_string(),
source,
})?;
Ok(())
}
pub fn load_tensor(path: &Path) -> Result<Tensor> {
let path_str = path.display().to_string();
let bytes = fs::read(path).map_err(|source| DataError::Io {
path: path_str.clone(),
source,
})?;
if bytes.len() < 9 || &bytes[..4] != MAGIC {
return Err(DataError::BadMagic { path: path_str }.into());
}
let dtype = tag_dtype(bytes[4])?;
let mut off = 5usize;
let read_u64 = |off: &mut usize| -> std::result::Result<u64, DataError> {
if *off + 8 > bytes.len() {
return Err(DataError::Truncated("tensor header".to_string()));
}
let v = u64::from_le_bytes(bytes[*off..*off + 8].try_into().unwrap());
*off += 8;
Ok(v)
};
let ndim = read_u64(&mut off)? as usize;
let mut shape = Vec::with_capacity(ndim);
for _ in 0..ndim {
shape.push(read_u64(&mut off)? as i64);
}
let numel: i64 = shape.iter().product();
let expected = (numel as usize) * elem_size(dtype);
if off + expected != bytes.len() {
return Err(DataError::SizeMismatch {
path: path_str,
expected,
found: bytes.len() - off,
}
.into());
}
Tensor::from_blob(&bytes[off..], &shape, dtype, Device::CPU)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum DataFormat {
Bin,
Csv,
Both,
}
pub fn save_dataset_as(dir: impl AsRef<Path>, ds: &Dataset, format: DataFormat) -> Result<()> {
let dir = dir.as_ref();
match format {
DataFormat::Bin => save_dataset(dir, ds),
DataFormat::Csv => save_csv_dataset(dir, &ds.inputs, &ds.targets),
DataFormat::Both => {
save_dataset(dir, ds)?;
save_csv_dataset(dir, &ds.inputs, &ds.targets)
}
}
}
pub fn load_dataset_auto(dir: impl AsRef<Path>) -> Result<Dataset> {
let dir = dir.as_ref();
if dir.join("inputs.bin").exists() {
return load_dataset(dir);
}
if dir.join("inputs.csv").exists() {
return load_csv_dataset(dir);
}
Err(DataError::Io {
path: dir.display().to_string(),
source: std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("no inputs.bin or inputs.csv found in {}", dir.display()),
),
}
.into())
}
pub fn save_dataset(dir: impl AsRef<Path>, ds: &Dataset) -> Result<()> {
let dir = dir.as_ref();
fs::create_dir_all(dir).map_err(|source| DataError::Io {
path: dir.display().to_string(),
source,
})?;
save_tensor(&dir.join("inputs.bin"), &ds.inputs)?;
save_tensor(&dir.join("targets.bin"), &ds.targets)?;
let meta = serde_json::json!({
"inputs_shape": ds.inputs.shape(),
"targets_shape": ds.targets.shape(),
});
let meta = serde_json::to_string_pretty(&meta)
.map_err(|e| DataError::Json(format!("meta.json: {e}")))?;
fs::write(dir.join("meta.json"), meta).map_err(|source| DataError::Io {
path: dir.join("meta.json").display().to_string(),
source,
})?;
Ok(())
}
pub fn load_dataset(dir: impl AsRef<Path>) -> Result<Dataset> {
let dir = dir.as_ref();
let inputs = load_tensor(&dir.join("inputs.bin"))?;
let targets = load_tensor(&dir.join("targets.bin"))?;
Ok(Dataset { inputs, targets })
}
pub fn resolve_dataset(dir: impl AsRef<Path>) -> Result<Dataset> {
resolve_inputs_targets_datasets(dir)
}
pub fn resolve_inputs_targets_datasets(dir: impl AsRef<Path>) -> Result<Dataset> {
let dir = dir.as_ref();
let cache_dir = dir.join("flodl_data");
if cache_dir.join("inputs.bin").exists() {
return load_dataset(&cache_dir);
}
if dir.join("inputs.bin").exists() {
return load_dataset(dir);
}
if dir.join("inputs.csv").exists() {
let dataset = load_csv_dataset(dir)?;
std::fs::create_dir_all(&cache_dir)
.map_err(|e| flodl::tensor::TensorError::new(&e.to_string()))?;
save_dataset(&cache_dir, &dataset)?;
return Ok(dataset);
}
Err(DataError::Io {
path: dir.display().to_string(),
source: std::io::Error::new(
std::io::ErrorKind::NotFound,
format!(
"no dataset found in {} — expected inputs.csv|bin + targets.csv|bin (or train/ + test/ subdirs for resolve_train_test_datasets)",
dir.display()
),
),
}
.into())
}
pub fn resolve_train_test_datasets(dir: impl AsRef<Path>) -> Result<(Dataset, Dataset)> {
resolve_two_splits(dir.as_ref(), "train", "test")
}
pub fn resolve_train_val_datasets(dir: impl AsRef<Path>) -> Result<(Dataset, Dataset)> {
resolve_two_splits(dir.as_ref(), "train", "val")
}
fn resolve_two_splits(dir: &Path, learn: &str, rank: &str) -> Result<(Dataset, Dataset)> {
let learn_dir = dir.join(learn);
let rank_dir = dir.join(rank);
for sub in [&learn_dir, &rank_dir] {
if !sub.exists() {
let name = sub.file_name().and_then(|s| s.to_str()).unwrap_or("?");
return Err(DataError::Io {
path: sub.display().to_string(),
source: std::io::Error::new(
std::io::ErrorKind::NotFound,
format!(
"missing '{name}/' split in {dir} — a two-role layout needs {dir}/{learn}/ and {dir}/{rank}/ each with inputs.csv|bin + targets.csv|bin",
dir = dir.display(),
),
),
}
.into());
}
}
let learn_ds = resolve_inputs_targets_datasets(&learn_dir)?;
let rank_ds = resolve_inputs_targets_datasets(&rank_dir)?;
let (li, ri) = (learn_ds.inputs.shape()[1], rank_ds.inputs.shape()[1]);
if li != ri {
return Err(DataError::Csv(format!(
"feature dim mismatch: {learn}/ inputs have {li} columns, {rank}/ inputs have {ri}"
))
.into());
}
let (lt, rt) = (learn_ds.targets.shape()[1], rank_ds.targets.shape()[1]);
if lt != rt {
return Err(DataError::Csv(format!(
"target dim mismatch: {learn}/ targets have {lt} columns, {rank}/ targets have {rt}"
))
.into());
}
Ok((learn_ds, rank_ds))
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PoolLayout {
Single,
TrainVal,
TrainTest,
}
impl PoolLayout {
pub fn eval_dir(self) -> Option<&'static str> {
match self {
Self::TrainVal => Some("val"),
Self::TrainTest => Some("test"),
Self::Single => None,
}
}
}
pub fn detect_pool_layout(dir: impl AsRef<Path>) -> Result<PoolLayout> {
let dir = dir.as_ref();
let has = |name: &str| dir.join(name).exists();
let (train, val, test, exam) = (has("train"), has("val"), has("test"), has("exam"));
match (train, val, test) {
(true, true, true) if exam => Err(DataError::Csv(format!(
"{dir} holds `val/`, `test/` AND `exam/`: with `val/` ranking, `test/` would already be the \
report-only exam, so the second exam store could never be read. Keep `train/` + `val/` + \
`test/` (test is the exam) or `train/` + `val/` + `exam/`, not all four",
dir = dir.display()
))
.into()),
(true, true, _) => Ok(PoolLayout::TrainVal),
(true, false, true) => Ok(PoolLayout::TrainTest),
(false, true, _) => Err(DataError::Csv(format!(
"{dir} has `val/` but no `train/` — the val pool RANKS, it never trains, so there is \
nothing to learn from",
dir = dir.display()
))
.into()),
(false, false, true) => Err(DataError::Csv(format!(
"{dir} has `test/` but no `train/` — add `train/`, or make this dir the pool itself \
(inputs.csv|bin + targets.csv|bin)",
dir = dir.display()
))
.into()),
_ => Ok(PoolLayout::Single),
}
}
pub fn exam_dir(dir: &Path, layout: PoolLayout) -> Option<std::path::PathBuf> {
let exam = dir.join("exam");
if exam.exists() {
return Some(exam);
}
match layout {
PoolLayout::TrainVal if dir.join("test").exists() => Some(dir.join("test")),
_ => None,
}
}
pub fn load_csv_dataset(dir: impl AsRef<Path>) -> Result<Dataset> {
let dir = dir.as_ref();
let inputs_path = dir.join("inputs.csv");
let targets_path = dir.join("targets.csv");
if !inputs_path.exists() {
return Err(DataError::Io {
path: inputs_path.display().to_string(),
source: std::io::Error::new(std::io::ErrorKind::NotFound, "inputs.csv not found"),
}
.into());
}
if !targets_path.exists() {
return Err(DataError::Io {
path: targets_path.display().to_string(),
source: std::io::Error::new(std::io::ErrorKind::NotFound, "targets.csv not found"),
}
.into());
}
let inputs_raw = std::fs::read_to_string(&inputs_path).map_err(|e| DataError::Io {
path: inputs_path.display().to_string(),
source: e,
})?;
let targets_raw = std::fs::read_to_string(&targets_path).map_err(|e| DataError::Io {
path: targets_path.display().to_string(),
source: e,
})?;
let mut inputs_flat: Vec<f32> = Vec::new();
let mut input_dim: Option<usize> = None;
let mut n_samples: usize = 0;
let mut skipped_header = false;
for line in inputs_raw.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let row: Vec<f32> = match line
.split(',')
.map(|s| s.trim().parse::<f32>())
.collect::<std::result::Result<Vec<_>, _>>()
{
Ok(row) => row,
Err(e) => {
if n_samples == 0 && !skipped_header {
skipped_header = true;
tracing::debug!("inputs.csv: skipping header row: {line}");
continue;
}
return Err(DataError::Csv(format!("inputs.csv parse error: {e}")).into());
}
};
if let Some(dim) = input_dim {
if row.len() != dim {
return Err(DataError::Csv(format!(
"inputs.csv: row {n_samples} has {} columns, expected {dim}",
row.len()
))
.into());
}
} else {
input_dim = Some(row.len());
}
inputs_flat.extend(row);
n_samples += 1;
}
let input_dim = input_dim.ok_or_else(|| DataError::Csv("inputs.csv is empty".into()))?;
let mut targets_flat: Vec<f32> = Vec::new();
let mut target_dim: Option<usize> = None;
let mut n_targets: usize = 0;
let mut skipped_header = false;
for line in targets_raw.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let row: Vec<f32> = match line
.split(',')
.map(|s| s.trim().parse::<f32>())
.collect::<std::result::Result<Vec<_>, _>>()
{
Ok(row) => row,
Err(e) => {
if n_targets == 0 && !skipped_header {
skipped_header = true;
tracing::debug!("targets.csv: skipping header row: {line}");
continue;
}
return Err(DataError::Csv(format!("targets.csv parse error: {e}")).into());
}
};
if let Some(dim) = target_dim {
if row.len() != dim {
return Err(DataError::Csv(format!(
"targets.csv: row {n_targets} has {} columns, expected {dim}",
row.len()
))
.into());
}
} else {
target_dim = Some(row.len());
}
targets_flat.extend(row);
n_targets += 1;
}
let target_dim = target_dim.ok_or_else(|| DataError::Csv("targets.csv is empty".into()))?;
if n_samples != n_targets {
return Err(DataError::Csv(format!(
"sample count mismatch: inputs has {n_samples}, targets has {n_targets}"
))
.into());
}
let inputs = Tensor::from_f32(
&inputs_flat,
&[n_samples as i64, input_dim as i64],
Device::CPU,
)?;
let targets = Tensor::from_f32(
&targets_flat,
&[n_targets as i64, target_dim as i64],
Device::CPU,
)?;
Ok(Dataset { inputs, targets })
}
pub fn save_csv_dataset(dir: impl AsRef<Path>, inputs: &Tensor, targets: &Tensor) -> Result<()> {
let dir = dir.as_ref();
fs::create_dir_all(dir).map_err(|source| DataError::Io {
path: dir.display().to_string(),
source,
})?;
let in_data = inputs
.to_f32_vec()
.map_err(|e| DataError::Csv(format!("inputs to_f32_vec: {e}")))?;
let tgt_data = targets
.to_f32_vec()
.map_err(|e| DataError::Csv(format!("targets to_f32_vec: {e}")))?;
let in_shape = inputs.shape();
let tgt_shape = targets.shape();
let (n_in, dim_in) = (in_shape[0] as usize, in_shape[1] as usize);
let (n_tgt, dim_tgt) = (tgt_shape[0] as usize, tgt_shape[1] as usize);
let mut inputs_csv = String::with_capacity(n_in * dim_in * 8);
for row in 0..n_in {
let start = row * dim_in;
let line: Vec<String> = (0..dim_in)
.map(|c| format!("{:.6}", in_data[start + c]))
.collect();
inputs_csv.push_str(&line.join(","));
inputs_csv.push('\n');
}
fs::write(dir.join("inputs.csv"), inputs_csv).map_err(|source| DataError::Io {
path: dir.join("inputs.csv").display().to_string(),
source,
})?;
let mut targets_csv = String::with_capacity(n_tgt * dim_tgt * 8);
for row in 0..n_tgt {
let start = row * dim_tgt;
let line: Vec<String> = (0..dim_tgt)
.map(|c| format!("{:.6}", tgt_data[start + c]))
.collect();
targets_csv.push_str(&line.join(","));
targets_csv.push('\n');
}
fs::write(dir.join("targets.csv"), targets_csv).map_err(|source| DataError::Io {
path: dir.join("targets.csv").display().to_string(),
source,
})?;
Ok(())
}
#[cfg(test)]
pub(crate) fn synthetic_classification(
n: usize,
in_dim: usize,
out_dim: usize,
seed: u64,
device: Device,
) -> Result<Dataset> {
let mut rng = fastrand::Rng::with_seed(seed);
let inputs: Vec<f32> = (0..n * in_dim).map(|_| rng.f32()).collect();
let inputs = Tensor::from_f32(&inputs, &[n as i64, in_dim as i64], device)?;
let mut targets = vec![0.0f32; n * out_dim];
for row in 0..n {
let c = rng.usize(0..out_dim);
targets[row * out_dim + c] = 1.0;
}
let targets = Tensor::from_f32(&targets, &[n as i64, out_dim as i64], device)?;
Ok(Dataset { inputs, targets })
}
pub fn one_hot(indices: &[usize], num_classes: usize, device: Device) -> Result<Tensor> {
let n = indices.len();
let mut data = vec![0.0f32; n * num_classes];
for (i, &c) in indices.iter().enumerate() {
data[i * num_classes + c] = 1.0;
}
Tensor::from_f32(&data, &[n as i64, num_classes as i64], device)
}
#[cfg(test)]
mod tests {
use super::*;
use proptest::prelude::*;
use std::path::PathBuf;
fn temp_dir() -> PathBuf {
std::env::temp_dir().join(format!("gras_data_test_{}", fastrand::u64(..)))
}
#[test]
fn test_tensor_roundtrip() {
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
let t = Tensor::from_f32(&[1.0, 2.0, 3.0, 4.0], &[2, 2], Device::CPU).unwrap();
let path = dir.join("t.bin");
save_tensor(&path, &t).unwrap();
let loaded = load_tensor(&path).unwrap();
assert_eq!(loaded.shape(), t.shape());
assert_eq!(loaded.to_f32_vec().unwrap(), t.to_f32_vec().unwrap());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn from_tensors_accepts_matching_rows_and_rejects_a_mismatch() {
let inputs = || Tensor::from_f32(&[1.0, 2.0, 3.0, 4.0], &[2, 2], Device::CPU).unwrap();
let targets = Tensor::from_f32(&[1.0, 0.0], &[2, 1], Device::CPU).unwrap();
let ds = Dataset::from_tensors(inputs(), targets).unwrap();
assert_eq!(ds.len(), 2);
assert_eq!(ds.inputs.shape(), &[2, 2]);
assert_eq!(ds.targets.shape(), &[2, 1]);
let short = Tensor::from_f32(&[1.0], &[1, 1], Device::CPU).unwrap();
let msg = Dataset::from_tensors(inputs(), short)
.unwrap_err()
.to_string();
assert!(
msg.contains("row count") && msg.contains('2') && msg.contains('1'),
"the mismatch error names both row counts, got: {msg}"
);
}
#[test]
fn test_dataset_roundtrip() {
let dir = temp_dir();
let inputs = Tensor::from_f32(&[1.0, 2.0, 3.0, 4.0], &[2, 2], Device::CPU).unwrap();
let targets = Tensor::from_f32(&[5.0, 6.0, 7.0, 8.0], &[2, 2], Device::CPU).unwrap();
let ds = Dataset { inputs, targets };
save_dataset(&dir, &ds).unwrap();
let loaded = load_dataset(&dir).unwrap();
assert_eq!(loaded.inputs.shape(), ds.inputs.shape());
assert_eq!(loaded.targets.shape(), ds.targets.shape());
assert_eq!(
loaded.inputs.to_f32_vec().unwrap(),
ds.inputs.to_f32_vec().unwrap()
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_load_tensor_rejects_garbage() {
let dir = temp_dir();
let path = dir.join("bad.bin");
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(&path, b"not a gras tensor").unwrap();
assert!(load_tensor(&path).is_err());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_csv_roundtrip() {
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("inputs.csv"), "1.0,2.0\n3.0,4.0\n5.0,6.0\n").unwrap();
std::fs::write(dir.join("targets.csv"), "0.0\n1.0\n0.0\n").unwrap();
let ds = load_csv_dataset(&dir).unwrap();
assert_eq!(ds.inputs.shape(), &[3, 2]);
assert_eq!(ds.targets.shape(), &[3, 1]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_csv_with_header() {
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("inputs.csv"), "feat1,feat2\n1.0,2.0\n3.0,4.0\n").unwrap();
std::fs::write(dir.join("targets.csv"), "label\n0.0\n1.0\n").unwrap();
let ds = load_csv_dataset(&dir).unwrap();
assert_eq!(ds.inputs.shape(), &[2, 2]);
assert_eq!(ds.targets.shape(), &[2, 1]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_csv_sample_count_mismatch() {
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("inputs.csv"), "1.0,2.0\n3.0,4.0\n5.0,6.0\n").unwrap();
std::fs::write(dir.join("targets.csv"), "0.0\n1.0\n").unwrap(); let result = load_csv_dataset(&dir);
assert!(result.is_err());
let err = result.unwrap_err().to_string();
assert!(err.contains("sample count mismatch"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_csv_skips_comments() {
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(
dir.join("inputs.csv"),
"# header comment\n1.0,2.0\n3.0,4.0\n",
)
.unwrap();
std::fs::write(dir.join("targets.csv"), "# another comment\n0.0\n1.0\n").unwrap();
let ds = load_csv_dataset(&dir).unwrap();
assert_eq!(ds.inputs.shape(), &[2, 2]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_save_dataset_as_roundtrip_all_formats() {
let mut rng = fastrand::Rng::with_seed(7);
let data: Vec<f32> = (0..24).map(|_| rng.f32()).collect();
let ds = Dataset {
inputs: Tensor::from_f32(&data, &[8, 3], Device::CPU).unwrap(),
targets: Tensor::from_f32(
&[0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0],
&[8, 1],
Device::CPU,
)
.unwrap(),
};
for fmt in [DataFormat::Bin, DataFormat::Csv, DataFormat::Both] {
let dir = std::env::temp_dir().join(format!("gras_data_fmt_{:?}", fmt));
let _ = std::fs::remove_dir_all(&dir);
save_dataset_as(&dir, &ds, fmt).unwrap();
let loaded = load_dataset_auto(&dir).unwrap();
assert_eq!(loaded.inputs.shape(), ds.inputs.shape());
assert_eq!(loaded.targets.shape(), ds.targets.shape());
let _ = std::fs::remove_dir_all(&dir);
}
let dir = std::env::temp_dir().join("gras_data_fmt_Both_files");
let _ = std::fs::remove_dir_all(&dir);
save_dataset_as(&dir, &ds, DataFormat::Both).unwrap();
assert!(dir.join("inputs.bin").exists(), "Bin part of Both");
assert!(dir.join("inputs.csv").exists(), "Csv part of Both");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn test_load_dataset_auto_prefers_bin() {
let mut rng = fastrand::Rng::with_seed(11);
let data: Vec<f32> = (0..16).map(|_| rng.f32()).collect();
let ds = Dataset {
inputs: Tensor::from_f32(&data, &[8, 2], Device::CPU).unwrap(),
targets: Tensor::from_f32(&data, &[8, 2], Device::CPU).unwrap(),
};
let dir = std::env::temp_dir().join("gras_data_auto_prefer");
let _ = std::fs::remove_dir_all(&dir);
save_dataset_as(&dir, &ds, DataFormat::Both).unwrap();
let loaded = load_dataset_auto(&dir).unwrap();
assert_eq!(loaded.inputs.to_f32_vec().unwrap(), data, "must load .bin");
let _ = std::fs::remove_dir_all(&dir);
}
proptest! {
#[test]
fn prop_tensor_roundtrip(
rows in 1usize..8,
cols in 1usize..8,
seed in 0u64..1_000_000,
) {
let mut rng = fastrand::Rng::with_seed(seed);
let data: Vec<f32> = (0..rows * cols).map(|_| rng.f32()).collect();
let t = Tensor::from_f32(&data, &[rows as i64, cols as i64], Device::CPU).unwrap();
let dir = temp_dir();
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("t.bin");
save_tensor(&path, &t).unwrap();
let loaded = load_tensor(&path).unwrap();
prop_assert_eq!(loaded.shape(), t.shape());
prop_assert_eq!(loaded.dtype(), t.dtype());
prop_assert_eq!(loaded.to_f32_vec().unwrap(), t.to_f32_vec().unwrap());
let _ = std::fs::remove_dir_all(&dir);
}
}
}