import pytest
from dataloader_rs import PyDataloader as DataLoader
from tests.py_dataloader_test_utils import ListDataset
def test_rejects_zero_batch_size():
with pytest.raises(ValueError, match="batch_size must be > 0"):
DataLoader(ListDataset([1, 2, 3]), batch_size=0)
def test_rejects_zero_prefetch_depth():
with pytest.raises(ValueError, match="prefetch_depth must be > 0"):
DataLoader(ListDataset([1, 2, 3]), prefetch_depth=0)
def test_rejects_shuffle_and_sampler_together():
with pytest.raises(ValueError, match="mutually exclusive"):
DataLoader(ListDataset([1, 2, 3]), shuffle=True, sampler=[0, 1, 2])
def test_rejects_dataset_not_subclassing_pydataset():
class PlainDataset:
def __len__(self):
return 2
def __getitem__(self, index):
return index
with pytest.raises(TypeError, match="must be a PyDataset, a PyAsyncDataset or a Rust dataset"):
DataLoader(PlainDataset(), batch_size=1)
def test_accepts_zero_num_workers():
loader = DataLoader(ListDataset(range(10)), batch_size=3, num_workers=0)
assert len(loader) == 4
def test_accepts_large_num_workers():
loader = DataLoader(ListDataset(range(10)), batch_size=3, num_workers=16)
assert len(loader) == 4
def test_accepts_large_prefetch_depth():
loader = DataLoader(ListDataset(range(10)), batch_size=3, prefetch_depth=1000)
assert len(loader) == 4
def test_batch_size_equals_dataset_len():
n = 7
loader = DataLoader(ListDataset(range(n)), batch_size=n)
batches = list(loader)
assert len(batches) == 1
assert sorted(batches[0]) == list(range(n))
def test_len_matches_drop_last_false():
loader = DataLoader(ListDataset(range(10)), batch_size=3, drop_last=False)
assert len(loader) == 4
def test_len_matches_drop_last_true():
loader = DataLoader(ListDataset(range(10)), batch_size=3, drop_last=True)
assert len(loader) == 3
def test_dataset_len_respected():
class SizedDs(ListDataset):
def __len__(self):
return 6
loader = DataLoader(SizedDs(range(10)), batch_size=3)
assert len(loader) == 2
def test_accepts_num_workers_argument():
loader = DataLoader(ListDataset(range(10)), batch_size=3, num_workers=0)
assert len(loader) == 4
def test_accepts_prefetch_depth_argument():
loader = DataLoader(ListDataset(range(10)), batch_size=3, prefetch_depth=4)
assert len(loader) == 4
def test_collate_fn_receives_list():
received_type = []
def inspecting_collate(items):
received_type.append(type(items))
return items
loader = DataLoader(ListDataset(range(4)), batch_size=4, collate_fn=inspecting_collate)
list(loader) assert received_type, "collate_fn must have been called at least once"
assert received_type[0] is list, f"collate_fn must receive a list, got {received_type[0]}"
def test_len_counts_sampler_indices_not_dataset():
loader = DataLoader(ListDataset(range(10)), batch_size=2, sampler=[0, 1, 2])
assert len(loader) == 2
assert len(list(loader)) == 2
def test_len_counts_sampler_indices_with_drop_last():
loader = DataLoader(ListDataset(range(10)), batch_size=2, sampler=[0, 1, 2], drop_last=True)
assert len(loader) == 1
assert len(list(loader)) == 1
class _IterOnlySampler:
def __iter__(self):
return iter([0, 1, 2])
class _BadLenSampler(_IterOnlySampler):
def __len__(self):
raise OSError("sampler index missing")
def test_len_raises_type_error_for_sampler_without_len():
loader = DataLoader(ListDataset(range(10)), batch_size=2, sampler=_IterOnlySampler())
with pytest.raises(TypeError):
len(loader)
assert list(loader) == [[0, 1], [2]]
def test_len_propagates_sampler_len_error():
loader = DataLoader(ListDataset(range(10)), batch_size=2, sampler=_BadLenSampler())
with pytest.raises(OSError, match="sampler index missing"):
len(loader)
def test_huge_prefetch_depth_works():
for depth in (10**12, 2**64 - 1):
loader = DataLoader(
ListDataset(range(8)), batch_size=2, num_workers=1, prefetch_depth=depth
)
assert list(loader) == [[0, 1], [2, 3], [4, 5], [6, 7]]
def test_rejects_thread_count_overflow():
with pytest.raises(ValueError, match="too large"):
DataLoader(ListDataset(range(4)), num_workers=2**63, item_concurrency=4)