import threading
import pytest
from dataloader_rs import PyDataloader as DataLoader
from dataloader_rs import PyDataset
class BatchedDs(PyDataset):
def __init__(self, n):
super().__init__()
self.n = n
self.calls = []
self._lock = threading.Lock()
def __len__(self):
return self.n
def __getitem__(self, index):
raise AssertionError("__getitem__ must not be called when __getitems__ exists")
def __getitems__(self, indices):
with self._lock:
self.calls.append(indices)
return [i * 10 for i in indices]
@pytest.mark.parametrize("num_workers", [0, 2])
def test_getitems_called_once_per_batch(num_workers):
ds = BatchedDs(10)
loader = DataLoader(ds, batch_size=4, num_workers=num_workers)
assert list(loader) == [[0, 10, 20, 30], [40, 50, 60, 70], [80, 90]]
assert sorted(ds.calls) == [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9]]
assert all(isinstance(call, list) for call in ds.calls)
@pytest.mark.parametrize("num_workers", [0, 2])
def test_getitems_output_goes_through_collate_fn(num_workers):
loader = DataLoader(BatchedDs(6), batch_size=3, num_workers=num_workers, collate_fn=sum)
assert list(loader) == [30, 120]
@pytest.mark.parametrize("num_workers", [0, 2])
def test_getitems_receives_sampler_order(num_workers):
ds = BatchedDs(6)
loader = DataLoader(ds, batch_size=3, num_workers=num_workers, sampler=[5, 3, 1, 0, 2, 4])
assert list(loader) == [[50, 30, 10], [0, 20, 40]]
@pytest.mark.parametrize("num_workers", [0, 2])
def test_getitems_may_return_any_iterable(num_workers):
class TupleDs(BatchedDs):
def __getitems__(self, indices):
return tuple(i + 1 for i in indices)
loader = DataLoader(TupleDs(4), batch_size=2, num_workers=num_workers)
assert list(loader) == [[1, 2], [3, 4]]
@pytest.mark.parametrize("num_workers", [0, 2])
def test_getitems_wrong_length_raises(num_workers):
class ShortDs(BatchedDs):
def __getitems__(self, indices):
return indices[:-1]
loader = DataLoader(ShortDs(4), batch_size=2, num_workers=num_workers)
with pytest.raises(ValueError, match="__getitems__ returned 1 samples for 2 indices"):
next(iter(loader))
@pytest.mark.parametrize("num_workers", [0, 2])
def test_getitems_exception_propagates(num_workers):
class FailingDs(BatchedDs):
def __getitems__(self, indices):
raise RuntimeError("batched read failed")
loader = DataLoader(FailingDs(4), batch_size=2, num_workers=num_workers)
with pytest.raises(RuntimeError, match="batched read failed"):
next(iter(loader))
@pytest.mark.parametrize("num_workers", [0, 2])
def test_getitems_set_to_none_falls_back_to_getitem(num_workers):
class OptOutDs(BatchedDs):
__getitems__ = None
def __getitem__(self, index):
return index + 100
loader = DataLoader(OptOutDs(3), batch_size=2, num_workers=num_workers)
assert list(loader) == [[100, 101], [102]]
@pytest.mark.parametrize("num_workers", [0, 2])
def test_datasets_without_getitems_still_use_getitem(num_workers):
class PlainDs(PyDataset):
def __len__(self):
return 3
def __getitem__(self, index):
return index
loader = DataLoader(PlainDs(), batch_size=2, num_workers=num_workers)
assert list(loader) == [[0, 1], [2]]