mod common;
use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array, Int64Array, RecordBatch};
use arrow_schema::{DataType, Field, Schema};
use common::Mp4;
use lecop::{
dataset::Dataset,
validate::{Checks, Issue, Limits, Severity, validate, validate_with, validate_with_checks},
};
use parquet::arrow::ArrowWriter;
use std::{fs, path::Path, sync::Arc};
const FPS: f32 = 10.0;
const DIM: i32 = 2;
struct Ep {
len: usize,
ts: Vec<f32>,
action: Vec<f32>,
with_video: bool,
with_data: bool,
range_to_delta: i64,
index_shift: i64,
video_delta: i32,
video_fps: u32,
video_garbage: bool,
video_size: (u16, u16),
video_sparse: bool,
task: i64,
}
fn v3_video_bytes(group: &[Ep]) -> Vec<u8> {
if group[0].video_garbage {
return video_bytes(0, &group[0]);
}
let mut m = Mp4::constant_fps(0, group[0].video_fps);
m.size = Some(group[0].video_size);
m.stts = group
.iter()
.map(|e| (e.len as u32, if e.video_sparse { 2048 } else { 1024 }))
.collect();
m.bytes()
}
fn video_bytes(frames: i64, e: &Ep) -> Vec<u8> {
if e.video_garbage {
return b"not an mp4".to_vec();
}
let mut m = Mp4::constant_fps(frames.max(0) as u32, e.video_fps);
m.size = Some(e.video_size);
m.bytes()
}
fn good(len: usize) -> Ep {
static CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let base = CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed) as f32 * 1000.0;
Ep {
len,
ts: (0..len).map(|i| i as f32 / FPS).collect(),
action: (0..len * DIM as usize)
.map(|i| {
let row = i / DIM as usize;
base + i as f32 * 0.1 + ((row * 7) % 5) as f32 * 0.05
})
.collect(),
with_video: true,
with_data: true,
range_to_delta: 0,
index_shift: 0,
video_delta: 0,
video_fps: FPS as u32,
video_garbage: false,
video_size: (4, 4),
video_sparse: false,
task: 0,
}
}
fn ramp(frames: usize) -> Vec<f32> {
(0..frames * DIM as usize)
.map(|i| 1.0 + i as f32 * 0.5)
.collect()
}
fn fsl(vals: Vec<f32>) -> ArrayRef {
let item = Arc::new(Field::new("item", DataType::Float32, true));
Arc::new(FixedSizeListArray::new(
item,
DIM,
Arc::new(Float32Array::from(vals)),
None,
))
}
fn write_ds(root: &Path, eps: &[Ep], claimed_total_frames: Option<usize>) {
for d in [
"meta",
"data/chunk-000",
"videos/chunk-000/observation.images.top",
] {
fs::create_dir_all(root.join(d)).unwrap();
}
let frames: usize = eps.iter().map(|e| e.len).sum();
let info = serde_json::json!({
"codebase_version": "v2.1", "robot_type": "test", "fps": FPS,
"total_episodes": eps.len(), "total_frames": claimed_total_frames.unwrap_or(frames),
"chunks_size": 1000,
"data_path": "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet",
"video_path": "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4",
"features": {
"action": {"dtype": "float32", "shape": [DIM]},
"observation.state": {"dtype": "float32", "shape": [DIM]},
"observation.images.top": {"dtype": "video", "shape": [4, 4, 3]},
"timestamp": {"dtype": "float32", "shape": [1]},
"frame_index": {"dtype": "int64", "shape": [1]},
"task_index": {"dtype": "int64", "shape": [1]},
}
});
fs::write(root.join("meta/info.json"), info.to_string()).unwrap();
common::write_stats(root);
fs::write(
root.join("meta/tasks.jsonl"),
"{\"task_index\": 0, \"task\": \"t\"}\n",
)
.unwrap();
let jsonl: String = eps
.iter()
.enumerate()
.map(|(i, e)| {
format!(
"{}\n",
serde_json::json!({"episode_index": i, "length": e.len, "tasks": ["t"]})
)
})
.collect();
fs::write(root.join("meta/episodes.jsonl"), jsonl).unwrap();
for (i, e) in eps.iter().enumerate() {
if e.with_video {
fs::write(
root.join(format!(
"videos/chunk-000/observation.images.top/episode_{i:06}.mp4"
)),
video_bytes(e.len as i64 + e.video_delta as i64, e),
)
.unwrap();
}
if !e.with_data {
continue;
}
let schema = Arc::new(Schema::new(vec![
Field::new("timestamp", DataType::Float32, false),
Field::new("frame_index", DataType::Int64, false),
Field::new("task_index", DataType::Int64, false),
Field::new(
"action",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
false,
),
Field::new(
"observation.state",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
false,
),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Float32Array::from(e.ts.clone())),
Arc::new(Int64Array::from((0..e.ts.len() as i64).collect::<Vec<_>>())),
i64s(vec![e.task; e.len]),
fsl(e.action.clone()),
fsl(ramp(e.len)),
],
)
.unwrap();
let mut w = ArrowWriter::try_new(
fs::File::create(root.join(format!("data/chunk-000/episode_{i:06}.parquet"))).unwrap(),
schema,
None,
)
.unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
}
}
fn run(eps: Vec<Ep>, claimed_frames: Option<usize>) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &eps, claimed_frames);
validate(&Dataset::open(dir.path()).unwrap()).unwrap()
}
fn codes(v: &[Issue]) -> Vec<&'static str> {
v.iter().map(|i| i.code).collect()
}
#[test]
fn healthy_dataset_has_no_issues() {
let v = run(vec![good(20), good(15)], None);
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn flags_timestamp_drift() {
let mut e = good(20);
for t in e.ts.iter_mut().skip(10) {
*t += 0.05;
} assert_eq!(codes(&run(vec![e], None)), ["timestamp_drift"]);
}
#[test]
fn flags_non_monotonic_timestamps() {
let mut e = good(20);
e.ts[7] = e.ts[5];
let c = codes(&run(vec![e], None));
assert!(c.contains(&"timestamp_not_monotonic"), "{c:?}");
}
#[test]
fn flags_nan_in_action() {
let mut e = good(20);
e.action[5] = f32::NAN;
assert_eq!(codes(&run(vec![e], None)), ["non_finite"]);
}
#[test]
fn warns_on_frozen_action() {
let mut e = good(20);
e.action = vec![0.5; 20 * DIM as usize];
let v = run(vec![e], None);
assert_eq!(codes(&v), ["action_constant"]);
assert_eq!(v[0].severity, Severity::Warning);
}
#[test]
fn flags_missing_video_and_data_files() {
let mut a = good(10);
a.with_video = false;
let mut b = good(10);
b.with_data = false;
let c = codes(&run(vec![a, b], None));
assert!(
c.contains(&"video_missing") && c.contains(&"data_missing"),
"{c:?}"
);
}
#[test]
fn flags_total_frames_mismatch() {
assert_eq!(codes(&run(vec![good(10)], Some(99))), ["frame_count"]);
}
#[test]
fn rejects_unknown_future_version() {
let dir = tempfile::tempdir().unwrap();
fs::create_dir_all(dir.path().join("meta")).unwrap();
fs::write(dir.path().join("meta/info.json"), r#"{"codebase_version":"v9.0","fps":30,"total_episodes":0,"total_frames":0,"data_path":"x","features":{}}"#).unwrap();
let err = Dataset::open(dir.path()).err().unwrap().to_string();
assert!(err.contains("v9.0"), "{err}");
}
use arrow_array::{Float64Array, StringArray};
const PER_FILE: usize = 2;
fn i64s(v: Vec<i64>) -> ArrayRef {
Arc::new(Int64Array::from(v))
}
fn write_ds_v3(root: &Path, eps: &[Ep]) {
let vk = "observation.images.top";
for d in [
"meta/episodes/chunk-000",
"data/chunk-000",
&format!("videos/{vk}/chunk-000"),
] {
fs::create_dir_all(root.join(d)).unwrap();
}
let frames: usize = eps.iter().map(|e| e.len).sum();
let info = serde_json::json!({
"codebase_version": "v3.0", "robot_type": "test", "fps": FPS,
"total_episodes": eps.len(), "total_frames": frames,
"data_path": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet",
"video_path": "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4",
"features": {
"action": {"dtype": "float32", "shape": [DIM]},
"observation.state": {"dtype": "float32", "shape": [DIM]},
"observation.images.top": {"dtype": "video", "shape": [4, 4, 3]},
"timestamp": {"dtype": "float32", "shape": [1]},
"frame_index": {"dtype": "int64", "shape": [1]},
"episode_index": {"dtype": "int64", "shape": [1]},
"index": {"dtype": "int64", "shape": [1]},
}
});
fs::write(root.join("meta/info.json"), info.to_string()).unwrap();
common::write_stats(root);
let (mut from, mut cols): (i64, [Vec<i64>; 4]) = (0, Default::default()); let mut file_idx = Vec::new();
let (mut vfrom, mut vto) = (Vec::<f64>::new(), Vec::<f64>::new());
for (g, group) in eps.chunks(PER_FILE).enumerate() {
if group[0].with_video {
fs::write(
root.join(format!("videos/{vk}/chunk-000/file-{g:03}.mp4")),
v3_video_bytes(group),
)
.unwrap();
}
let mut rows = (
Vec::<f32>::new(),
Vec::<i64>::new(),
Vec::<i64>::new(),
Vec::<i64>::new(),
Vec::<f32>::new(),
);
let mut t = 0.0;
for (j, e) in group.iter().enumerate() {
let i = g * PER_FILE + j;
vfrom.push(t);
vto.push(t + (e.len as i64 + e.video_delta as i64) as f64 / FPS as f64);
t += e.len as f64 / FPS as f64;
cols[0].push(i as i64);
cols[1].push(e.len as i64);
cols[2].push(from);
cols[3].push(from + e.len as i64 + e.range_to_delta);
file_idx.push(g as i64);
rows.0.extend(&e.ts);
rows.1.extend(0..e.ts.len() as i64);
rows.2.extend(std::iter::repeat_n(i as i64, e.ts.len()));
rows.3
.extend(from + e.index_shift..from + e.index_shift + e.ts.len() as i64);
rows.4.extend(&e.action);
from += e.len as i64;
}
if !group[0].with_data {
continue;
}
let schema = Arc::new(Schema::new(vec![
Field::new("timestamp", DataType::Float32, false),
Field::new("frame_index", DataType::Int64, false),
Field::new("episode_index", DataType::Int64, false),
Field::new("index", DataType::Int64, false),
Field::new(
"action",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
false,
),
Field::new(
"observation.state",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
false,
),
]));
let frames = rows.0.len();
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Float32Array::from(rows.0)),
i64s(rows.1),
i64s(rows.2),
i64s(rows.3),
fsl(rows.4),
fsl(ramp(frames)),
],
)
.unwrap();
let f = fs::File::create(root.join(format!("data/chunk-000/file-{g:03}.parquet"))).unwrap();
let mut w = ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
}
let n = eps.len();
let zeros = || i64s(vec![0; n]);
let names = [
"episode_index",
"length",
"dataset_from_index",
"dataset_to_index",
"data/chunk_index",
"data/file_index",
"videos/observation.images.top/chunk_index",
"videos/observation.images.top/file_index",
"videos/observation.images.top/from_timestamp",
"videos/observation.images.top/to_timestamp",
"meta/episodes/chunk_index",
"meta/episodes/file_index",
];
let arrays: Vec<ArrayRef> = vec![
i64s(cols[0].clone()),
i64s(cols[1].clone()),
i64s(cols[2].clone()),
i64s(cols[3].clone()),
zeros(),
i64s(file_idx.clone()),
zeros(),
i64s(file_idx),
Arc::new(Float64Array::from(vfrom)),
Arc::new(Float64Array::from(vto)),
zeros(),
zeros(),
];
let fields: Vec<Field> = names
.iter()
.zip(&arrays)
.map(|(n, a)| Field::new(*n, a.data_type().clone(), false))
.collect();
let _ = StringArray::from(vec!["unused"; 0]);
let schema = Arc::new(Schema::new(fields));
let batch = RecordBatch::try_new(schema.clone(), arrays).unwrap();
let f = fs::File::create(root.join("meta/episodes/chunk-000/file-000.parquet")).unwrap();
let mut w = ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
}
fn run_v3(eps: Vec<Ep>) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds_v3(dir.path(), &eps);
validate(&Dataset::open(dir.path()).unwrap()).unwrap()
}
#[test]
fn v3_healthy_packed_dataset_has_no_issues() {
let v = run_v3(vec![good(20), good(15), good(10), good(12), good(18)]);
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn v3_flags_timestamp_drift_in_second_episode_of_a_file() {
let mut e = good(20);
for t in e.ts.iter_mut().skip(5) {
*t += 0.05;
}
let v = run_v3(vec![good(10), e, good(10)]);
assert_eq!(codes(&v), ["timestamp_drift"]);
assert_eq!(v[0].episode, Some(1));
}
fn action_stats(rows: &[&[f32]]) -> serde_json::Value {
let n = rows.len() as f64;
let (mut mn, mut mx, mut mean, mut std) = (vec![], vec![], vec![], vec![]);
for j in 0..DIM as usize {
let c: Vec<f64> = rows.iter().map(|r| r[j] as f64).collect();
let m = c.iter().sum::<f64>() / n;
mn.push(c.iter().cloned().fold(f64::INFINITY, f64::min));
mx.push(c.iter().cloned().fold(f64::NEG_INFINITY, f64::max));
std.push((c.iter().map(|x| (x - m).powi(2)).sum::<f64>() / n).sqrt());
mean.push(m);
}
serde_json::json!({"min": mn, "max": mx, "mean": mean, "std": std, "count": [rows.len()]})
}
fn v3_with_action_stats(edit: impl Fn(&mut serde_json::Value)) -> Vec<Issue> {
let eps = vec![good(20), good(15), good(10)];
let dir = tempfile::tempdir().unwrap();
write_ds_v3(dir.path(), &eps);
let rows: Vec<&[f32]> = eps
.iter()
.flat_map(|e| e.action.chunks(DIM as usize))
.collect();
let path = dir.path().join("meta/stats.json");
let mut stats: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();
stats["action"] = action_stats(&rows);
edit(&mut stats["action"]);
fs::write(&path, stats.to_string()).unwrap();
validate(&Dataset::open(dir.path()).unwrap()).unwrap()
}
#[test]
fn v3_statistics_that_match_the_data_report_nothing() {
let v = v3_with_action_stats(|_| {});
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn v3_flags_each_stale_statistic_and_names_feature_dimension_and_values() {
for (stat, dim) in [("min", 0), ("max", 1), ("mean", 1), ("std", 0)] {
let v = v3_with_action_stats(|a| a[stat][dim] = 123456.0.into());
let m = v
.iter()
.find(|i| i.code == "stats_mismatch")
.unwrap_or_else(|| panic!("{stat}: {v:?}"));
assert_eq!((m.severity, m.episode), (Severity::Warning, None));
assert!(
m.message.contains("action")
&& m.message.contains(&format!("dimension {dim} {stat}"))
&& m.message.contains("123456"),
"{}",
m.message
);
}
}
#[test]
fn v3_counts_the_further_differences_in_the_singular_and_the_plural() {
let one = v3_with_action_stats(|a| {
a["mean"][0] = 123456.0.into();
a["std"][0] = 123456.0.into();
});
let two = v3_with_action_stats(|a| {
a["mean"][0] = 123456.0.into();
a["std"][0] = 123456.0.into();
a["max"][0] = 123456.0.into();
});
let msg = |v: &[Issue]| {
v.iter()
.find(|i| i.code == "stats_mismatch")
.unwrap()
.message
.clone()
};
assert!(
msg(&one).contains("(and 1 more difference)"),
"{}",
msg(&one)
);
assert!(
msg(&two).contains("(and 2 more differences)"),
"{}",
msg(&two)
);
}
#[test]
fn v3_flags_a_stale_frame_count_in_the_statistics() {
let v = v3_with_action_stats(|a| a["count"] = serde_json::json!([999]));
let m = v
.iter()
.find(|i| i.code == "stats_mismatch")
.unwrap_or_else(|| panic!("{v:?}"));
assert!(
m.message.contains("count") && m.message.contains("999") && m.message.contains("45"),
"{}",
m.message
);
}
fn v21_stats_line(
root: &Path,
index: usize,
ep: &Ep,
edit: impl Fn(&mut serde_json::Value),
) -> String {
let text = fs::read_to_string(stats_path(root)).unwrap();
let mut line: serde_json::Value = serde_json::from_str(text.lines().next().unwrap()).unwrap();
let rows: Vec<&[f32]> = ep.action.chunks(DIM as usize).collect();
line["episode_index"] = index.into();
line["stats"]["action"] = action_stats(&rows);
edit(&mut line["stats"]["action"]);
line.to_string()
}
fn v21_with_action_stats(edit: impl Fn(usize, &mut serde_json::Value)) -> Vec<Issue> {
let eps = vec![good(20), good(15), good(10)];
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &eps, None);
let lines: Vec<String> = eps
.iter()
.enumerate()
.map(|(i, e)| v21_stats_line(dir.path(), i, e, |a| edit(i, a)))
.collect();
fs::write(stats_path(dir.path()), lines.join("\n") + "\n").unwrap();
validate(&Dataset::open(dir.path()).unwrap()).unwrap()
}
#[test]
fn v21_statistics_that_match_each_episode_report_nothing() {
let v = v21_with_action_stats(|_, _| {});
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn v21_flags_the_episode_whose_statistics_are_stale_and_names_the_values() {
for (stat, dim) in [("min", 0), ("max", 1), ("mean", 1), ("std", 0)] {
let v = v21_with_action_stats(|ep, a| {
if ep == 1 {
a[stat][dim] = 123456.0.into();
}
});
let found: Vec<&Issue> = v.iter().filter(|i| i.code == "stats_mismatch").collect();
assert_eq!(found.len(), 1, "{stat}: {v:?}");
let m = found[0];
assert_eq!((m.severity, m.episode), (Severity::Warning, Some(1)));
assert!(
m.message.contains("action")
&& m.message.contains(&format!("dimension {dim} {stat}"))
&& m.message.contains("123456")
&& m.message.contains("episodes_stats.jsonl"),
"{}",
m.message
);
}
}
#[test]
fn v21_flags_a_stale_frame_count_of_one_episode() {
let v = v21_with_action_stats(|ep, a| {
if ep == 2 {
a["count"] = serde_json::json!([999]);
}
});
let m = v
.iter()
.find(|i| i.code == "stats_mismatch")
.unwrap_or_else(|| panic!("{v:?}"));
assert_eq!(m.episode, Some(2));
assert!(
m.message.contains("count") && m.message.contains("999") && m.message.contains("10"),
"{}",
m.message
);
}
#[test]
fn v21_does_not_compare_an_episode_that_has_no_statistics_entry() {
let eps = vec![good(20), good(15)];
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &eps, None);
let line = v21_stats_line(dir.path(), 0, &eps[0], |_| {});
fs::write(stats_path(dir.path()), format!("{line}\n")).unwrap();
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn v3_names_the_file_and_the_frame_where_the_global_index_breaks() {
let mut e = good(10);
e.index_shift = 1000;
let v = run_v3(vec![good(10), e, good(10)]);
let m = v
.iter()
.find(|i| i.code == "index_mismatch")
.unwrap_or_else(|| panic!("{v:?}"));
assert_eq!((m.severity, m.episode), (Severity::Error, Some(1)));
assert!(
m.message.contains("file-000.parquet")
&& m.message.contains("from 10")
&& m.message.contains("frame 0 reads 1010 instead of 10"),
"{}",
m.message
);
}
#[test]
fn v3_flags_range_not_matching_length() {
let mut e = good(10);
e.range_to_delta = 1;
let c = codes(&run_v3(vec![good(10), e, good(10)]));
assert!(c.contains(&"range_length"), "{c:?}");
}
#[test]
fn v3_flags_missing_data_and_video_files() {
let mut a = good(10);
a.with_data = false;
let mut b = good(10);
b.with_video = false;
let c = codes(&run_v3(vec![a, good(10), b, good(10)]));
assert!(
c.contains(&"data_missing") && c.contains(&"video_missing"),
"{c:?}"
);
}
#[test]
fn flags_video_frame_count_mismatch() {
let mut e = good(20);
e.video_delta = -3;
let v = run(vec![e], None);
assert_eq!(codes(&v), ["video_frame_count"]);
assert_eq!(v[0].severity, Severity::Error);
}
#[test]
fn warns_on_video_fps_mismatch() {
let mut e = good(20);
e.video_fps = 15; let v = run(vec![e], None);
assert_eq!(codes(&v), ["video_fps"]);
assert_eq!(v[0].severity, Severity::Warning);
}
#[test]
fn flags_unreadable_video() {
let mut e = good(20);
e.video_garbage = true;
let v = run(vec![e], None);
assert_eq!(codes(&v), ["video_unreadable"]);
}
#[test]
fn v3_flags_span_not_matching_length_and_beyond_end() {
let mut e = good(10);
e.video_delta = 5; let c = codes(&run_v3(vec![good(10), e, good(10)]));
assert!(c.contains(&"video_span_length"), "{c:?}");
assert!(c.contains(&"video_span_beyond_end"), "{c:?}"); }
#[test]
fn v3_flags_episodes_whose_span_holds_the_wrong_number_of_frames() {
let mut sparse = good(10);
sparse.video_sparse = true; let v = run_v3(vec![good(10), sparse]); let bad: Vec<_> = v
.iter()
.filter(|i| i.code == "video_frame_count")
.map(|i| i.episode)
.collect();
assert_eq!(bad, [Some(1)], "{v:?}");
}
#[test]
fn v3_healthy_packed_dataset_has_no_frame_count_findings() {
let v = run_v3(vec![good(10), good(12), good(17)]);
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn v3_reports_a_shared_unreadable_video_once() {
let mut a = good(10);
a.video_garbage = true; let v = run_v3(vec![a, good(10), good(10)]);
assert_eq!(codes(&v), ["video_unreadable"]);
}
fn run_with_data_file(batch: RecordBatch) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(10)], None);
let f = fs::File::create(dir.path().join("data/chunk-000/episode_000000.parquet")).unwrap();
let mut w = ArrowWriter::try_new(f, batch.schema(), None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
validate(&Dataset::open(dir.path()).unwrap()).unwrap()
}
#[test]
fn reports_a_missing_timestamp_column_instead_of_panicking() {
let schema = Arc::new(Schema::new(vec![
Field::new("frame_index", DataType::Int64, false),
Field::new(
"action",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
false,
),
]));
let batch = RecordBatch::try_new(
schema,
vec![i64s((0..10).collect()), fsl(vec![0.5; 10 * DIM as usize])],
)
.unwrap();
let c = codes(&run_with_data_file(batch));
assert!(c.contains(&"timestamp_missing"), "{c:?}");
}
#[test]
fn reports_ragged_list_columns_instead_of_panicking() {
use arrow_array::{ListArray, types::Float32Type};
let rows = (0..10).map(|i| Some(vec![Some(i as f32); if i == 0 { 2 } else { 1 }]));
let action: ArrayRef = Arc::new(ListArray::from_iter_primitive::<Float32Type, _, _>(rows));
let schema = Arc::new(Schema::new(vec![
Field::new("timestamp", DataType::Float32, false),
Field::new("action", action.data_type().clone(), true),
]));
let ts: Vec<f32> = (0..10).map(|i| i as f32 / FPS).collect();
let batch =
RecordBatch::try_new(schema, vec![Arc::new(Float32Array::from(ts)), action]).unwrap();
let c = codes(&run_with_data_file(batch));
assert!(c.contains(&"column_shape"), "{c:?}");
}
#[test]
fn rejects_non_positive_fps() {
let dir = tempfile::tempdir().unwrap();
fs::create_dir_all(dir.path().join("meta")).unwrap();
fs::write(
dir.path().join("meta/info.json"),
r#"{"codebase_version":"v2.1","fps":0,"total_episodes":0,"total_frames":0,"data_path":"x","features":{}}"#,
)
.unwrap();
let err = Dataset::open(dir.path()).err().unwrap().to_string();
assert!(err.contains("fps"), "{err}");
}
fn edit_info(root: &Path, f: impl FnOnce(&mut serde_json::Value)) {
let p = root.join("meta/info.json");
let mut v: serde_json::Value = serde_json::from_slice(&fs::read(&p).unwrap()).unwrap();
f(&mut v);
fs::write(p, v.to_string()).unwrap();
common::write_stats(root);
}
fn rename_feature(root: &Path, from: &str, to: &str) {
edit_info(root, |v| {
let f = v["features"].as_object_mut().unwrap();
let spec = f.remove(from).unwrap();
f.insert(to.into(), spec);
});
for chunk in fs::read_dir(root.join("data")).unwrap() {
for file in fs::read_dir(chunk.unwrap().path()).unwrap() {
let path = file.unwrap().path();
let reader = parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder::try_new(
fs::File::open(&path).unwrap(),
)
.unwrap()
.build()
.unwrap();
let batches: Vec<RecordBatch> = reader.map(|b| b.unwrap()).collect();
let fields: Vec<Field> = batches[0]
.schema()
.fields()
.iter()
.map(|f| match f.name() == from {
true => f.as_ref().clone().with_name(to),
false => f.as_ref().clone(),
})
.collect();
let schema = Arc::new(Schema::new(fields));
let mut w =
ArrowWriter::try_new(fs::File::create(&path).unwrap(), schema.clone(), None)
.unwrap();
for b in batches {
w.write(&RecordBatch::try_new(schema.clone(), b.columns().to_vec()).unwrap())
.unwrap();
}
w.close().unwrap();
}
}
}
fn rename_action_feature(root: &Path) {
edit_info(root, |v| {
let f = v["features"].as_object_mut().unwrap();
let a = f.remove("action").unwrap();
f.insert("actions".into(), a);
});
}
#[test]
fn warns_when_there_is_no_standard_action_feature() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
rename_feature(dir.path(), "action", "actions");
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["nonstandard_features"]);
assert_eq!(v[0].severity, Severity::Warning);
assert_eq!(v[0].episode, None);
}
#[test]
fn action_checks_follow_an_actions_column() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(10)], None);
rename_action_feature(dir.path());
let schema = Arc::new(Schema::new(vec![
Field::new("timestamp", DataType::Float32, false),
Field::new(
"actions",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
false,
),
]));
let ts: Vec<f32> = (0..10).map(|i| i as f32 / FPS).collect();
let batch = RecordBatch::try_new(
schema,
vec![
Arc::new(Float32Array::from(ts)),
fsl(vec![0.5; 10 * DIM as usize]),
],
)
.unwrap();
let f = fs::File::create(dir.path().join("data/chunk-000/episode_000000.parquet")).unwrap();
let mut w = ArrowWriter::try_new(f, batch.schema(), None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
let c = codes(&validate(&Dataset::open(dir.path()).unwrap()).unwrap());
assert!(c.contains(&"action_constant"), "{c:?}");
}
fn idle_then_moving(len: usize, idle_rows: usize) -> Ep {
let mut e = good(len);
e.action = (0..len)
.flat_map(|i| {
let v = if i < idle_rows { 0.5 } else { i as f32 * 0.1 };
[v, v]
})
.collect();
e
}
#[test]
fn warns_on_mostly_idle_episode_and_says_when_it_starts_moving() {
let v = run(vec![idle_then_moving(20, 15)], None); assert_eq!(codes(&v), ["idle_episode"]);
assert_eq!(v[0].severity, Severity::Warning);
assert!(v[0].message.contains("frame 15"), "{}", v[0].message);
}
#[test]
fn does_not_warn_on_a_short_pause() {
let v = run(vec![idle_then_moving(20, 6)], None); assert!(v.is_empty(), "{v:?}");
}
fn moving_with_hold(len: usize, from: usize, rows: usize) -> Ep {
let mut e = good(len);
e.action = (0..len)
.flat_map(|i| {
let steps = match i {
i if i < from => i,
i if i < from + rows => from,
i => i - (rows - 1),
};
let v = steps as f32 * 0.1;
[v, v]
})
.collect();
e
}
fn run_with_frozen_seconds(eps: Vec<Ep>, seconds: Option<f64>) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &eps, None);
let checks = Checks {
frozen_seconds: seconds,
..Checks::default()
};
validate_with_checks(&Dataset::open(dir.path()).unwrap(), 1, &checks).unwrap()
}
#[test]
fn warns_when_the_action_is_held_for_three_seconds_in_the_middle_and_says_where() {
let v = run(vec![moving_with_hold(100, 20, 35)], None);
assert_eq!(codes(&v), ["action_frozen"], "{v:?}");
assert_eq!((v[0].severity, v[0].episode), (Severity::Warning, Some(0)));
assert!(
v[0].message.contains("3.4 s") && v[0].message.contains("frame 20"),
"{}",
v[0].message
);
}
#[test]
fn does_not_warn_on_a_hold_shorter_than_three_seconds() {
let v = run(vec![moving_with_hold(100, 20, 26)], None); assert!(v.is_empty(), "{v:?}");
}
#[test]
fn does_not_warn_on_rest_at_the_start_of_the_episode() {
let v = run(vec![moving_with_hold(100, 0, 40)], None);
assert!(!codes(&v).contains(&"action_frozen"), "{v:?}");
}
#[test]
fn the_hold_length_that_is_reported_follows_frozen_seconds() {
let hold = || vec![moving_with_hold(100, 20, 26)]; assert_eq!(
codes(&run_with_frozen_seconds(hold(), Some(2.0))),
["action_frozen"]
);
assert!(run_with_frozen_seconds(hold(), Some(3.0)).is_empty());
let long = || vec![moving_with_hold(100, 20, 35)]; assert!(run_with_frozen_seconds(long(), Some(5.0)).is_empty());
}
#[test]
fn does_not_mistake_a_sibling_feature_for_the_action() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
rename_feature(dir.path(), "action", "action_mask");
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["nonstandard_features"]);
assert!(!v[0].message.contains("action_mask"), "{}", v[0].message);
}
#[test]
fn report_is_identical_for_any_thread_count() {
use common::synth::{Synth, write_v2, write_v3};
let s = Synth {
episodes: 40,
frames: 50,
dim: 3,
fps: 30,
per_file: 7,
};
for v3 in [false, true] {
let dir = tempfile::tempdir().unwrap();
if v3 {
write_v3(dir.path(), &s);
} else {
write_v2(dir.path(), &s);
}
edit_info(dir.path(), |v| {
v["fps"] = 31.into();
v["total_frames"] = 1.into();
});
let ds = Dataset::open(dir.path()).unwrap();
let one = format!("{:?}", validate_with(&ds, 1).unwrap());
assert!(
one.contains("timestamp_drift"),
"fixture must have findings"
);
for jobs in [2, 8] {
assert_eq!(
one,
format!("{:?}", validate_with(&ds, jobs).unwrap()),
"v3={v3} jobs={jobs}"
);
}
}
}
fn schema_issues_after(edit: impl FnOnce(&mut serde_json::Value)) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20), good(15)], None);
edit_info(dir.path(), edit);
validate(&Dataset::open(dir.path()).unwrap()).unwrap()
}
#[test]
fn flags_a_declared_feature_missing_from_the_data_once() {
let v = schema_issues_after(|v| {
v["features"]["observation.extra"] = serde_json::json!({"dtype": "float32", "shape": [3]});
});
assert_eq!(codes(&v), ["feature_missing"], "{v:?}");
assert_eq!(v[0].severity, Severity::Error);
assert!(v[0].message.contains("observation.extra"), "{v:?}");
}
#[test]
fn flags_a_column_whose_dtype_differs_from_info_json() {
let v = schema_issues_after(|v| {
v["features"]["action"]["dtype"] = "int64".into();
});
assert_eq!(codes(&v), ["feature_dtype"], "{v:?}");
}
#[test]
fn flags_a_column_whose_width_differs_from_info_json() {
let v = schema_issues_after(|v| {
v["features"]["action"]["shape"] = serde_json::json!([DIM + 1]);
});
assert_eq!(codes(&v), ["feature_shape"], "{v:?}");
}
#[test]
fn warns_on_a_data_column_info_json_does_not_declare() {
let v = schema_issues_after(|v| {
v["features"].as_object_mut().unwrap().remove("frame_index");
});
assert_eq!(codes(&v), ["feature_extra"], "{v:?}");
assert_eq!(v[0].severity, Severity::Warning);
}
#[test]
fn warns_on_an_episode_shorter_than_a_second() {
let v = run(vec![good(20), good(3)], None);
let short: Vec<_> = v.iter().filter(|i| i.code == "episode_short").collect();
assert_eq!(short.len(), 1, "{v:?}");
assert_eq!(short[0].episode, Some(1));
assert_eq!(short[0].severity, Severity::Warning);
}
#[test]
fn flags_an_empty_episode() {
let v = run(vec![good(20), good(0)], None);
assert!(codes(&v).contains(&"episode_empty"), "{v:?}");
assert!(!codes(&v).contains(&"episode_short"), "{v:?}");
}
#[test]
fn a_single_jump_is_not_pluralised() {
let mut eps: Vec<Ep> = (0..8).map(|_| good(20)).collect();
for x in &mut eps[3].action[10 * DIM as usize..] {
*x += 1000.0;
}
let v = run(eps, None);
let j = v
.iter()
.find(|i| i.code == "action_jump")
.unwrap_or_else(|| panic!("{v:?}"));
assert!(
j.message.starts_with("1 frame-to-frame jump in `action`"),
"{}",
j.message
);
}
#[test]
fn warns_on_an_episode_far_longer_than_the_others() {
let mut eps: Vec<Ep> = (0..20).map(|i| good(20 + i % 4)).collect();
eps.push(good(400));
let v = run(eps, None);
let out: Vec<_> = v
.iter()
.filter(|i| i.code == "episode_length_outlier")
.collect();
assert_eq!(out.len(), 1, "{v:?}");
assert_eq!(out[0].episode, Some(20));
assert_eq!(out[0].severity, Severity::Warning);
assert!(
out[0].message.contains("longer than the other episodes"),
"{}",
out[0].message
);
}
#[test]
fn does_not_call_a_few_episodes_outliers() {
let v = run(vec![good(20), good(21), good(400)], None);
assert!(!codes(&v).contains(&"episode_length_outlier"), "{v:?}");
}
#[test]
fn warns_on_an_episode_that_repeats_an_earlier_one() {
let first = good(20);
let mut copy = good(20);
copy.action = first.action.clone();
let v = run(vec![first, good(15), copy], None);
let dup: Vec<_> = v.iter().filter(|i| i.code == "duplicate_episode").collect();
assert_eq!(dup.len(), 1, "{v:?}");
assert_eq!(dup[0].episode, Some(2));
assert!(dup[0].message.contains("episode 0"), "{}", dup[0].message);
assert_eq!(dup[0].severity, Severity::Warning);
}
#[test]
fn does_not_flag_same_length_episodes_with_different_actions() {
let first = good(20);
let mut other = good(20);
other.action = first.action.clone();
other.action[5] += 1.0;
let v = run(vec![first, other], None);
assert!(!codes(&v).contains(&"duplicate_episode"), "{v:?}");
}
fn action_column_run(f: impl Fn(usize, usize) -> f32) -> Vec<Issue> {
let mut e = good(150);
e.action = (0..150)
.flat_map(|row| (0..DIM as usize).map(move |d| (row, d)))
.map(|(row, d)| f(row, d))
.collect();
run(vec![e], None)
}
#[test]
fn a_varying_action_has_no_numeric_findings() {
let v = action_column_run(|row, d| row as f32 * 0.01 + d as f32);
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn warns_on_a_dimension_that_never_changes() {
let v = action_column_run(|row, d| if d == 0 { row as f32 * 0.01 } else { 0.5 });
let f: Vec<_> = v
.iter()
.filter(|i| i.code == "constant_dimension")
.collect();
assert_eq!(f.len(), 1, "{v:?}");
assert_eq!(f[0].severity, Severity::Warning);
assert!(
f[0].message.contains("action") && f[0].message.contains('1'),
"{}",
f[0].message
);
}
#[test]
fn warns_on_a_dimension_stuck_at_its_maximum() {
let v = action_column_run(|row, d| match (d, row) {
(0, _) => row as f32 * 0.01,
(_, 3) => 4.0,
_ => 5.0,
});
assert!(codes(&v).contains(&"clipped_dimension"), "{v:?}");
}
#[test]
fn warns_on_a_value_far_from_the_rest() {
let v = action_column_run(|row, d| match (d, row) {
(1, 70) => 1000.0,
_ => row as f32 * 0.01 + d as f32,
});
let f: Vec<_> = v.iter().filter(|i| i.code == "outlier_values").collect();
assert_eq!(f.len(), 1, "{v:?}");
}
#[test]
fn flags_a_video_whose_size_differs_from_info_json_once() {
let mut a = good(20);
a.video_size = (8, 6);
let mut b = good(15);
b.video_size = (8, 6);
let v = run(vec![a, b, good(12)], None);
let r: Vec<_> = v.iter().filter(|i| i.code == "video_resolution").collect();
assert_eq!(r.len(), 1, "{v:?}");
assert!(
r[0].message.contains("8x6") && r[0].message.contains("4x4"),
"{}",
r[0].message
);
}
#[test]
fn flags_the_same_in_a_packed_v3_dataset() {
let mut a = good(20);
a.video_size = (8, 6);
let v = run_v3(vec![a, good(15)]);
assert_eq!(codes(&v), ["video_resolution"], "{v:?}");
}
#[test]
fn reads_channel_first_shapes_from_the_feature_names() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
edit_info(dir.path(), |v| {
let f = &mut v["features"]["observation.images.top"];
f["shape"] = serde_json::json!([3, 4, 4]);
f["names"] = serde_json::json!(["channels", "height", "width"]);
});
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert!(v.is_empty(), "{v:?}");
}
fn write_tasks_parquet(root: &Path, n: usize) {
let schema = Arc::new(Schema::new(vec![Field::new(
"task_index",
DataType::Int64,
false,
)]));
let col: ArrayRef = i64s((0..n as i64).collect());
let batch = RecordBatch::try_new(schema.clone(), vec![col]).unwrap();
let f = fs::File::create(root.join("meta/tasks.parquet")).unwrap();
let mut w = ArrowWriter::try_new(f, schema, None).unwrap();
w.write(&batch).unwrap();
w.close().unwrap();
}
fn run_with_tasks(declared: usize, jsonl_lines: Option<usize>) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
edit_info(dir.path(), |v| v["total_tasks"] = declared.into());
if let Some(n) = jsonl_lines {
let lines: Vec<String> = (0..n)
.map(|i| format!("{{\"task_index\": {i}, \"task\": \"t{i}\"}}"))
.collect();
fs::write(dir.path().join("meta/tasks.jsonl"), lines.join("\n")).unwrap();
} else {
fs::remove_file(dir.path().join("meta/tasks.jsonl")).unwrap();
}
validate(&Dataset::open(dir.path()).unwrap()).unwrap()
}
#[test]
fn flags_declared_tasks_without_task_metadata() {
let v = run_with_tasks(1, None);
assert_eq!(codes(&v), ["tasks_missing"], "{v:?}");
assert_eq!(v[0].severity, Severity::Error);
}
#[test]
fn warns_when_total_tasks_disagrees_with_the_task_file() {
let v = run_with_tasks(1, Some(2));
assert_eq!(codes(&v), ["tasks_count"], "{v:?}");
assert_eq!(v[0].severity, Severity::Warning);
}
#[test]
fn matching_task_file_is_clean() {
let v = run_with_tasks(2, Some(2));
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn v3_reads_the_task_count_from_tasks_parquet() {
for (rows, want) in [(1, vec![]), (3, vec!["tasks_count"])] {
let dir = tempfile::tempdir().unwrap();
write_ds_v3(dir.path(), &[good(20)]);
edit_info(dir.path(), |v| v["total_tasks"] = 1.into());
write_tasks_parquet(dir.path(), rows);
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), want, "{v:?}");
}
let dir = tempfile::tempdir().unwrap();
write_ds_v3(dir.path(), &[good(20)]);
edit_info(dir.path(), |v| v["total_tasks"] = 1.into());
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["tasks_missing"], "{v:?}");
}
fn stats_path(root: &Path) -> std::path::PathBuf {
root.join("meta/episodes_stats.jsonl")
}
#[test]
fn warns_when_the_statistics_file_is_missing() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
fs::remove_file(stats_path(dir.path())).unwrap();
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["stats_missing"], "{v:?}");
assert_eq!(v[0].severity, Severity::Warning);
}
#[test]
fn warns_when_a_feature_has_no_statistics() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
fs::write(
stats_path(dir.path()),
"{\"episode_index\":0,\"stats\":{\"action\":{}}}\n",
)
.unwrap();
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["stats_feature_missing"], "{v:?}");
assert!(v[0].message.contains("timestamp"), "{}", v[0].message);
assert!(!v[0].message.contains("action"), "{}", v[0].message);
}
#[test]
fn v3_reads_stats_json() {
let dir = tempfile::tempdir().unwrap();
write_ds_v3(dir.path(), &[good(20)]);
let p = dir.path().join("meta/stats.json");
fs::write(&p, r#"{"action": {}}"#).unwrap();
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["stats_feature_missing"], "{v:?}");
fs::write(&p, "not json").unwrap();
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["stats_missing"], "{v:?}");
}
#[test]
fn flags_a_task_index_missing_from_the_task_metadata() {
let mut a = good(20);
a.task = 3;
let mut b = good(15);
b.task = 7;
let v = run(vec![good(12), a, b], None);
assert_eq!(codes(&v), ["task_index"], "{v:?}");
assert_eq!(v[0].severity, Severity::Error);
assert!(v[0].message.contains("3, 7"), "{}", v[0].message);
}
#[test]
fn warns_when_there_is_no_observation_state() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
rename_feature(dir.path(), "observation.state", "observation.joints");
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["nonstandard_features"], "{v:?}");
assert!(
v[0].message.contains("observation.state"),
"{}",
v[0].message
);
}
#[test]
fn names_every_missing_info_field_at_once() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
edit_info(dir.path(), |v| {
let o = v.as_object_mut().unwrap();
o.remove("fps");
o.remove("total_frames");
});
let err = Dataset::open(dir.path()).err().unwrap().to_string();
assert!(
err.contains("missing required fields: fps, total_frames"),
"{err}"
);
}
#[test]
fn v2_0_datasets_use_stats_json_and_are_otherwise_checked_like_v2_1() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20), good(15)], None);
fs::remove_file(stats_path(dir.path())).unwrap();
edit_info(dir.path(), |v| v["codebase_version"] = "v2.0".into());
assert!(dir.path().join("meta/stats.json").exists());
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert!(v.is_empty(), "{v:?}");
fs::remove_file(dir.path().join("meta/stats.json")).unwrap();
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["stats_missing"], "{v:?}");
}
#[test]
fn flags_an_episode_whose_action_jumps_and_names_it() {
let mut eps: Vec<Ep> = (0..8).map(|_| good(20)).collect();
eps[3].action[2 * DIM as usize] += 1000.0;
eps[3].action[2 * DIM as usize + 1] += 1000.0;
let v = run(eps, None);
let j: Vec<_> = v.iter().filter(|i| i.code == "action_jump").collect();
assert_eq!(j.len(), 1, "{v:?}");
assert_eq!((j[0].episode, j[0].severity), (Some(3), Severity::Warning));
assert!(j[0].message.starts_with("2 "), "{}", j[0].message);
}
#[test]
fn flags_action_jumps_in_a_packed_v3_dataset() {
let mut eps: Vec<Ep> = (0..8).map(|_| good(20)).collect();
eps[6].action[5 * DIM as usize] += 1000.0;
eps[6].action[5 * DIM as usize + 1] += 1000.0;
let v = run_v3(eps);
let j: Vec<_> = v.iter().filter(|i| i.code == "action_jump").collect();
assert_eq!(j.len(), 1, "{v:?}");
assert_eq!(j[0].episode, Some(6));
}
#[test]
fn lists_at_most_five_jumping_episodes_and_counts_the_rest() {
let mut eps: Vec<Ep> = (0..40).map(|_| good(20)).collect();
for e in eps.iter_mut().take(7) {
for a in e.action.iter_mut().skip(2 * DIM as usize) {
*a += 1000.0;
}
}
let v = run(eps, None);
let j: Vec<_> = v.iter().filter(|i| i.code == "action_jump").collect();
assert_eq!(j.len(), 6, "{v:?}");
let summary = j.iter().find(|i| i.episode.is_none()).unwrap();
assert!(summary.message.contains("2 more"), "{}", summary.message);
}
fn drifting(n: usize, shift: f32) -> Vec<Ep> {
(0..n)
.map(|i| {
let mut e = good(20);
let off = if i >= n - n / 4 { shift } else { 0.0 };
e.action = (0..20 * DIM as usize)
.map(|k| k as f32 * 0.1 + off)
.collect();
e.action
.iter_mut()
.enumerate()
.for_each(|(k, a)| *a += (i % 2) as f32 * 0.01 * k as f32);
e
})
.collect()
}
#[test]
fn flags_a_shift_between_the_first_and_last_quarter_of_episodes() {
let v = run(drifting(12, 50.0), None);
let s: Vec<_> = v
.iter()
.filter(|i| i.code == "distribution_shift")
.collect();
assert_eq!(s.len(), 1, "{v:?}");
assert!(s[0].message.starts_with("action: "), "{}", s[0].message);
}
#[test]
fn no_shift_without_a_drift_and_none_under_ten_episodes() {
let v = run(drifting(12, 0.0), None);
assert!(v.iter().all(|i| i.code != "distribution_shift"), "{v:?}");
let v = run(drifting(9, 50.0), None);
assert!(v.iter().all(|i| i.code != "distribution_shift"), "{v:?}");
}
#[test]
fn warns_on_an_absolute_data_path_template() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
let abs = format!(
"{}/data/chunk-{{episode_chunk:03d}}/episode_{{episode_index:06d}}.parquet",
dir.path().display()
);
edit_info(dir.path(), |v| v["data_path"] = abs.into());
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["path_absolute"], "{v:?}");
assert_eq!(v[0].severity, Severity::Warning);
}
#[test]
fn warns_on_non_parquet_files_in_the_data_directory() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
fs::write(dir.path().join("data/chunk-000/.DS_Store"), b"x").unwrap();
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["data_stray_file"], "{v:?}");
assert_eq!(v[0].severity, Severity::Warning);
assert!(v[0].message.contains(".DS_Store"), "{v:?}");
}
#[test]
fn lists_five_stray_files_and_counts_the_rest() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
for i in 0..7 {
fs::write(dir.path().join(format!("data/chunk-000/x{i}.tmp")), b"x").unwrap();
}
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["data_stray_file"], "{v:?}");
assert!(v[0].message.ends_with("and 2 more"), "{v:?}");
}
fn run_limits(limits: serde_json::Value) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
let limits: Limits = serde_json::from_value(serde_json::json!({ "action": limits })).unwrap();
validate_with_checks(
&Dataset::open(dir.path()).unwrap(),
1,
&Checks {
limits,
..Checks::default()
},
)
.unwrap()
}
#[test]
fn limits_that_hold_report_nothing() {
let v = run_limits(serde_json::json!({"min": -1e9, "max": 1e9, "max_step": 100.0}));
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn flags_a_value_beyond_a_scalar_bound() {
let v = run_limits(serde_json::json!({"max": -1e9}));
assert_eq!(codes(&v), ["limit_out_of_range"], "{v:?}");
assert_eq!(v[0].severity, Severity::Error);
assert!(v[0].message.starts_with("action: dimension 0"), "{v:?}");
}
#[test]
fn per_dimension_bounds_flag_only_the_dimension_that_breaks_them() {
let v = run_limits(serde_json::json!({"min": [1e9, -1e9]}));
assert_eq!(codes(&v), ["limit_out_of_range"], "{v:?}");
assert!(
v[0].message.contains("dimension 0") && !v[0].message.contains("dimension 1"),
"{v:?}"
);
}
#[test]
fn flags_a_bound_list_of_the_wrong_width() {
let v = run_limits(serde_json::json!({"min": [0.0, 0.0, 0.0]}));
assert_eq!(codes(&v), ["limit_width"], "{v:?}");
}
#[test]
fn flags_the_episode_whose_frame_to_frame_change_exceeds_max_step() {
let v = run_limits(serde_json::json!({"max_step": 0.001}));
assert_eq!(codes(&v), ["limit_step"], "{v:?}");
assert_eq!(v[0].episode, Some(0));
}
#[cfg(feature = "decode")]
fn decode_run(replace_video: Option<&[(u8, u8)]>) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(40)], None);
if let Some(frames) = replace_video {
let p = dir
.path()
.join("videos/chunk-000/observation.images.top/episode_000000.mp4");
common::h264::video(&p, frames, None);
}
validate_with_checks(
&Dataset::open(dir.path()).unwrap(),
1,
&Checks {
decode: true,
..Checks::default()
},
)
.unwrap()
}
#[cfg(feature = "decode")]
#[test]
fn decode_reports_a_frozen_camera_on_its_episode() {
let mut frames = vec![(100, 3); 5];
frames.extend(vec![(100, 0); 35]);
let v = decode_run(Some(&frames));
let f = v.iter().find(|i| i.code == "video_frozen").expect("frozen");
assert_eq!(
(f.episode, f.severity),
(Some(0), Severity::Warning),
"{v:?}"
);
assert!(f.message.contains("episode_000000.mp4"), "{v:?}");
}
#[cfg(feature = "decode")]
#[test]
fn decode_says_when_a_video_cannot_be_decoded_at_all() {
let v = decode_run(None);
let f = v
.iter()
.find(|i| i.code == "video_decode")
.expect("not decoded");
assert_eq!(f.severity, Severity::Warning, "{v:?}");
assert!(f.message.contains("not decoded"), "{v:?}");
}
#[cfg(unix)]
fn symlink_data_file(ds_root: &Path, blobs: &Path) {
fs::create_dir_all(blobs).unwrap();
let file = ds_root.join("data/chunk-000/episode_000000.parquet");
let blob = blobs.join("blob");
fs::rename(&file, &blob).unwrap();
std::os::unix::fs::symlink(&blob, &file).unwrap();
}
#[cfg(unix)]
#[test]
fn warns_on_a_symlink_inside_the_dataset() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
symlink_data_file(dir.path(), &dir.path().join("elsewhere"));
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
let c = codes(&v);
assert!(c.contains(&"symlink"), "{v:?}");
let s = v.iter().find(|i| i.code == "symlink").unwrap();
assert_eq!(s.severity, Severity::Warning);
assert!(s.message.contains("episode_000000.parquet"), "{v:?}");
}
#[cfg(unix)]
#[test]
fn does_not_warn_on_symlinks_in_a_hub_cache_snapshot() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("datasets--org--name/snapshots/abc123");
write_ds(&root, &[good(20)], None);
symlink_data_file(&root, &dir.path().join("datasets--org--name/blobs"));
let v = validate(&Dataset::open(&root).unwrap()).unwrap();
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn warns_on_a_file_over_5_gb() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20)], None);
fs::File::create(dir.path().join("meta/big.bin"))
.unwrap()
.set_len(6 << 30)
.unwrap();
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert_eq!(codes(&v), ["file_oversized"], "{v:?}");
assert!(
v[0].message.replace('\\', "/").contains("meta/big.bin"),
"{v:?}"
);
}
fn v3_with_action_std(std: serde_json::Value) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds_v3(dir.path(), &[good(20)]);
let path = dir.path().join("meta/stats.json");
let mut stats: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();
stats["action"] = serde_json::json!({ "std": std });
fs::write(&path, stats.to_string()).unwrap();
let mut v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
v.retain(|i| i.code != "stats_mismatch");
v
}
#[test]
fn a_statistics_file_that_is_not_an_object_is_reported_once_and_not_judged_for_std() {
let dir = tempfile::tempdir().unwrap();
write_ds_v3(dir.path(), &[good(20)]);
fs::write(dir.path().join("meta/stats.json"), "[1, 2]").unwrap();
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert!(!codes(&v).contains(&"stats_zero_std"), "{v:?}");
assert!(
v.iter().any(|i| i.message.contains("no statistics object")),
"{v:?}"
);
}
#[test]
fn warns_on_a_dimension_with_zero_std_in_the_statistics() {
let v = v3_with_action_std(serde_json::json!([0.0, 1.0]));
assert_eq!(codes(&v), ["stats_zero_std"], "{v:?}");
assert_eq!(v[0].severity, Severity::Warning);
assert!(v[0].message.starts_with("action: dimension 0"), "{v:?}");
}
#[test]
fn a_statistics_file_with_positive_std_is_clean() {
let v = v3_with_action_std(serde_json::json!([0.5, 1.0]));
assert!(v.is_empty(), "{v:?}");
}
fn run_chunk(chunk: Option<usize>) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20), good(30), good(120)], None);
validate_with_checks(
&Dataset::open(dir.path()).unwrap(),
1,
&Checks {
chunk_size: chunk,
..Checks::default()
},
)
.unwrap()
}
#[test]
fn warns_when_episodes_are_shorter_than_the_chunk_size() {
let v = run_chunk(Some(100));
assert_eq!(codes(&v), ["episodes_below_chunk"], "{v:?}");
assert_eq!(v[0].severity, Severity::Warning);
assert!(
v[0].message
.starts_with("2 of 3 episodes have fewer than 100 frames"),
"{v:?}"
);
}
#[test]
fn no_chunk_size_means_no_chunk_check() {
assert!(run_chunk(None).is_empty());
assert!(run_chunk(Some(10)).is_empty());
}
fn v2_with_edit(edit: impl FnOnce(&Path)) -> Vec<Issue> {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(10), good(10)], None);
edit(dir.path());
validate(&Dataset::open(dir.path()).unwrap()).unwrap()
}
#[test]
fn flags_metadata_length_that_differs_from_the_data_file() {
let v = v2_with_edit(|root| {
let rows = r#"{"episode_index": 0, "length": 12, "tasks": ["t"]}
{"episode_index": 1, "length": 10, "tasks": ["t"]}
"#;
fs::write(root.join("meta/episodes.jsonl"), rows).unwrap();
});
let m = v
.iter()
.find(|i| i.code == "length_mismatch")
.expect("{v:?}");
assert_eq!((m.severity, m.episode), (Severity::Error, Some(0)));
assert!(
m.message.contains("12") && m.message.contains("10"),
"{v:?}"
);
assert!(m.message.contains("episode_000000.parquet"), "{v:?}");
}
#[test]
fn flags_total_episodes_that_disagrees_with_the_metadata() {
let v = v2_with_edit(|root| edit_info(root, |i| i["total_episodes"] = 5.into()));
assert_eq!(codes(&v), ["episode_count"], "{v:?}");
}
#[test]
fn flags_an_episode_whose_index_is_not_its_position() {
let v = v2_with_edit(|root| {
let rows = r#"{"episode_index": 0, "length": 10, "tasks": ["t"]}
{"episode_index": 4, "length": 10, "tasks": ["t"]}
"#;
fs::write(root.join("meta/episodes.jsonl"), rows).unwrap();
});
let m = v.iter().find(|i| i.code == "episode_index").expect("{v:?}");
assert!(m.message.contains("position 1"), "{v:?}");
}
#[test]
fn flags_a_scalar_column_declared_wider_in_info_json() {
let v = v2_with_edit(|root| {
edit_info(root, |i| {
i["features"]["timestamp"]["shape"] = serde_json::json!([3])
});
});
let m = v.iter().find(|i| i.code == "feature_shape").expect("{v:?}");
assert!(m.message.contains("`timestamp`"), "{v:?}");
}
#[test]
fn reports_a_data_file_that_is_not_parquet() {
let v = v2_with_edit(|root| {
fs::write(
root.join("data/chunk-000/episode_000001.parquet"),
b"not parquet",
)
.unwrap();
});
let m = v
.iter()
.find(|i| i.code == "data_unreadable")
.expect("{v:?}");
assert_eq!(m.episode, Some(1));
}
fn single_episode_batch(timestamps: ArrayRef, frame_index: Vec<i64>) -> RecordBatch {
let float = timestamps.data_type().clone();
let schema = Arc::new(Schema::new(vec![
Field::new("timestamp", float, false),
Field::new("frame_index", DataType::Int64, false),
Field::new("task_index", DataType::Int64, false),
Field::new(
"action",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
false,
),
Field::new(
"observation.state",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM),
false,
),
]));
let e = good(10);
RecordBatch::try_new(
schema,
vec![
timestamps,
Arc::new(Int64Array::from(frame_index)),
i64s(vec![0; 10]),
fsl(e.action),
fsl(ramp(10)),
],
)
.unwrap()
}
#[test]
fn flags_a_frame_index_that_does_not_run_from_zero() {
let ts: Vec<f32> = (0..10).map(|i| i as f32 / FPS).collect();
let batch = single_episode_batch(Arc::new(Float32Array::from(ts)), (1..=10).collect());
let v = run_with_data_file(batch);
assert!(codes(&v).contains(&"frame_index"), "{v:?}");
}
#[test]
fn reads_float64_timestamps() {
let ts: Vec<f64> = (0..10).map(|i| i as f64 / FPS as f64).collect();
let batch = single_episode_batch(Arc::new(Float64Array::from(ts)), (0..10).collect());
let v = run_with_data_file(batch);
assert_eq!(codes(&v), ["feature_dtype"], "{v:?}");
}
#[test]
fn limit_step_counts_the_other_episodes_that_break_it() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(20), good(20), good(20)], None);
let limits: Limits =
serde_json::from_value(serde_json::json!({ "action": {"max_step": 0.001} })).unwrap();
let checks = Checks {
limits,
..Checks::default()
};
let v = validate_with_checks(&Dataset::open(dir.path()).unwrap(), 1, &checks).unwrap();
assert_eq!(codes(&v), ["limit_step"], "{v:?}");
assert!(v[0].message.contains("and 2 more episodes"), "{v:?}");
}
fn run_clipped(fraction: Option<f64>) -> Vec<Issue> {
let mut e = good(200);
let cap = e.action[160 * DIM as usize];
for row in 160..200 {
e.action[row * DIM as usize] = cap;
}
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[e], None);
validate_with_checks(
&Dataset::open(dir.path()).unwrap(),
1,
&Checks {
clipped_fraction: fraction,
..Checks::default()
},
)
.unwrap()
}
#[test]
fn warns_on_a_dimension_with_5_percent_of_its_values_at_an_extreme() {
let v = run_clipped(None);
assert_eq!(codes(&v), ["clipped_dimension"], "{v:?}");
assert!(v[0].message.starts_with("action: dimension 0"), "{v:?}");
assert!(v[0].message.contains("20.0%"), "{v:?}");
}
#[test]
fn a_higher_clipped_fraction_lets_it_pass() {
assert!(run_clipped(Some(0.5)).is_empty());
}
#[test]
fn a_lower_clipped_fraction_still_flags_it() {
assert_eq!(codes(&run_clipped(Some(0.1))), ["clipped_dimension"]);
}
#[test]
fn fix_recomputes_the_task_total_from_the_task_metadata() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(10)], None);
edit_info(dir.path(), |i| i["total_tasks"] = 5.into());
let ds = Dataset::open(dir.path()).unwrap();
let issues = validate(&ds).unwrap();
assert_eq!(codes(&issues), ["tasks_count"], "{issues:?}");
let fixes = lecop::fix::plan(&ds, &issues, lecop::validate::task_count(&ds));
assert_eq!(
fixes,
[lecop::fix::Fix {
key: "total_tasks",
from: 5,
to: 1,
code: "tasks_count"
}]
);
lecop::fix::apply(dir.path(), &fixes).unwrap();
let v = validate(&Dataset::open(dir.path()).unwrap()).unwrap();
assert!(v.is_empty(), "{v:?}");
}
#[test]
fn fix_plans_nothing_when_a_total_is_right() {
let dir = tempfile::tempdir().unwrap();
write_ds(dir.path(), &[good(10)], None);
let ds = Dataset::open(dir.path()).unwrap();
assert!(lecop::fix::plan(&ds, &validate(&ds).unwrap(), None).is_empty());
}