use std::hash::{Hash, Hasher};
use std::path::{Path, PathBuf};
use arrow::array::{ArrayRef, RecordBatch, UInt32Array};
use arrow::datatypes::DataType;
use arrow::row::{RowConverter, SortField};
use crate::{Value, WindowOperator, WindowRow};
type Mk = Box<dyn Fn() -> Box<dyn WindowOperator + Send> + Send>;
const HOT_COMBINED: usize = u32::MAX as usize;
enum ShardSlot {
Hot(Box<dyn WindowOperator + Send>),
Frozen { path: PathBuf },
}
pub struct ShardedWindow {
shards: Vec<ShardSlot>,
group_by: Vec<String>,
converter: Option<RowConverter>,
mk: Mk,
dir: Option<PathBuf>,
budget: usize,
touch: Vec<u64>,
clock: u64,
frozen_seq: u64,
partitions: Vec<i32>,
now_ms: i64,
}
impl ShardedWindow {
pub fn new(
k: usize,
mk: impl Fn() -> Box<dyn WindowOperator + Send> + Send + 'static,
group_by: Vec<String>,
) -> Self {
let k = k.max(1);
let mk: Mk = Box::new(mk);
ShardedWindow {
shards: (0..k).map(|_| ShardSlot::Hot(mk())).collect(),
group_by,
converter: None,
mk,
dir: None,
budget: 0,
touch: vec![0; k],
clock: 0,
frozen_seq: 0,
partitions: Vec::new(),
now_ms: 0,
}
}
pub fn with_eviction(mut self, dir: PathBuf, budget_bytes: usize) -> Self {
let _ = std::fs::create_dir_all(&dir);
self.dir = Some(dir);
self.budget = budget_bytes;
self
}
pub fn shard_count(&self) -> usize {
self.shards.len()
}
pub fn resident_shards(&self) -> usize {
self.shards.iter().filter(|s| matches!(s, ShardSlot::Hot(_))).count()
}
fn bump(&mut self, i: usize) {
self.clock += 1;
self.touch[i] = self.clock;
}
fn hot(&mut self, i: usize) -> &mut Box<dyn WindowOperator + Send> {
if let ShardSlot::Frozen { path, .. } = &self.shards[i] {
let path = path.clone();
let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("thaw shard {i}: {e}"));
let mut op = (self.mk)();
op.load_snapshot(&bytes)
.unwrap_or_else(|e| panic!("thaw shard {i}: {e}"));
op.expect_partitions(&self.partitions, self.now_ms);
let _ = std::fs::remove_file(&path);
self.shards[i] = ShardSlot::Hot(op);
}
self.bump(i);
match &mut self.shards[i] {
ShardSlot::Hot(op) => op,
ShardSlot::Frozen { .. } => unreachable!("just thawed"),
}
}
fn resident_bytes(&self) -> usize {
self.shards
.iter()
.filter_map(|s| match s {
ShardSlot::Hot(op) => Some(op.state_bytes()),
ShardSlot::Frozen { .. } => None,
})
.sum()
}
fn freeze(&mut self, i: usize) -> bool {
let Some(dir) = self.dir.clone() else { return false };
let ShardSlot::Hot(op) = &mut self.shards[i] else {
return false;
};
if op.open_windows() != 0 {
return false;
}
let snap = op.snapshot();
self.frozen_seq += 1;
let path = dir.join(format!("shard-{i}-{}.arrow", self.frozen_seq));
if std::fs::write(&path, &snap).is_err() {
return false;
}
self.shards[i] = ShardSlot::Frozen { path };
true
}
fn enforce_budget(&mut self) {
if self.budget == 0 || self.dir.is_none() {
return;
}
while self.resident_bytes() > self.budget {
let victim = self
.shards
.iter()
.enumerate()
.filter(|(_, s)| matches!(s, ShardSlot::Hot(op) if op.open_windows() == 0))
.min_by_key(|(i, _)| self.touch[*i])
.map(|(i, _)| i);
match victim {
Some(i) if self.freeze(i) => {}
_ => break, }
}
}
fn shard_of_rows(&mut self, batch: &RecordBatch) -> Vec<usize> {
let k = self.shards.len();
if self.group_by.is_empty() || k == 1 {
return vec![0; batch.num_rows()];
}
let cols: Vec<ArrayRef> = self
.group_by
.iter()
.map(|g| match batch.column_by_name(g) {
Some(c) => std::sync::Arc::clone(c),
None => arrow::array::new_null_array(&DataType::Null, batch.num_rows()),
})
.collect();
if self.converter.is_none() {
let fields = cols.iter().map(|c| SortField::new(c.data_type().clone())).collect();
self.converter = Some(RowConverter::new(fields).expect("row converter"));
}
let rows = self
.converter
.as_mut()
.expect("converter")
.convert_columns(&cols)
.expect("group columns convert");
(0..batch.num_rows())
.map(|i| {
let mut h = std::collections::hash_map::DefaultHasher::new();
rows.row(i).as_ref().hash(&mut h);
(h.finish() as usize) % k
})
.collect()
}
fn split(&mut self, batch: &RecordBatch, event_ms: &[i64]) -> Vec<Option<(RecordBatch, Vec<i64>)>> {
let k = self.shards.len();
let of = self.shard_of_rows(batch);
let mut idx: Vec<Vec<u32>> = vec![Vec::new(); k];
let mut times: Vec<Vec<i64>> = vec![Vec::new(); k];
for (i, &s) in of.iter().enumerate() {
idx[s].push(i as u32);
times[s].push(event_ms[i]);
}
idx.into_iter()
.zip(times)
.map(|(rows, ts)| {
if rows.is_empty() {
None
} else {
let take = UInt32Array::from(rows);
let cols: Vec<ArrayRef> = batch
.columns()
.iter()
.map(|c| arrow::compute::take(c, &take, None).expect("take"))
.collect();
let sub = RecordBatch::try_new(batch.schema(), cols).expect("shard batch");
Some((sub, ts))
}
})
.collect()
}
}
fn concat(per_shard: &[Option<Option<RecordBatch>>]) -> Option<Option<RecordBatch>> {
let mut batches: Vec<RecordBatch> = Vec::new();
let mut any_batch_path = false;
for inner in per_shard.iter().flatten() {
any_batch_path = true;
if let Some(b) = inner {
if b.num_rows() > 0 {
batches.push(b.clone());
}
}
}
if !any_batch_path {
return None;
}
match batches.first() {
None => Some(None),
Some(first) => {
let schema = first.schema();
Some(Some(
arrow::compute::concat_batches(&schema, &batches).expect("concat fired"),
))
}
}
}
impl WindowOperator for ShardedWindow {
fn expect_partitions(&mut self, partitions: &[i32], now_ms: i64) {
self.partitions = partitions.to_vec();
self.now_ms = now_ms;
for s in &mut self.shards {
if let ShardSlot::Hot(op) = s {
op.expect_partitions(partitions, now_ms);
}
}
}
fn push(&mut self, partition: i32, event_ms: i64, row: &[(String, Value)]) -> bool {
let k = self.shards.len();
let s = if self.group_by.is_empty() || k == 1 {
0
} else {
let mut h = std::collections::hash_map::DefaultHasher::new();
for g in &self.group_by {
if let Some((_, v)) = row.iter().find(|(name, _)| name == g) {
format!("{v:?}").hash(&mut h);
}
}
(h.finish() as usize) % k
};
let out = self.hot(s).push(partition, event_ms, row);
self.enforce_budget();
out
}
fn push_batch(&mut self, partition: i32, batch: &RecordBatch, event_ms: &[i64]) -> Option<usize> {
let parts = self.split(batch, event_ms);
let mut late = 0usize;
let mut answered = false;
for (s, part) in parts.into_iter().enumerate() {
if let Some((sub, ts)) = part {
if let Some(l) = self.hot(s).push_batch(partition, &sub, &ts) {
answered = true;
late += l;
}
}
}
self.enforce_budget();
answered.then_some(late)
}
fn advance(&mut self, now_ms: i64) -> Vec<WindowRow> {
let out: Vec<WindowRow> = self
.shards
.iter_mut()
.filter_map(|s| match s {
ShardSlot::Hot(op) => Some(op.advance(now_ms)),
ShardSlot::Frozen { .. } => None, })
.flatten()
.collect();
self.enforce_budget();
out
}
fn advance_to(&mut self, watermark_ms: i64) -> Vec<WindowRow> {
let out: Vec<WindowRow> = self
.shards
.iter_mut()
.filter_map(|s| match s {
ShardSlot::Hot(op) => Some(op.advance_to(watermark_ms)),
ShardSlot::Frozen { .. } => None,
})
.flatten()
.collect();
self.enforce_budget();
out
}
fn advance_batch(&mut self, now_ms: i64) -> Option<Option<RecordBatch>> {
let per: Vec<_> = self
.shards
.iter_mut()
.map(|s| match s {
ShardSlot::Hot(op) => op.advance_batch(now_ms),
ShardSlot::Frozen { .. } => Some(None),
})
.collect();
let out = concat(&per);
self.enforce_budget();
out
}
fn advance_batch_to(&mut self, watermark_ms: i64) -> Option<Option<RecordBatch>> {
let per: Vec<_> = self
.shards
.iter_mut()
.map(|s| match s {
ShardSlot::Hot(op) => op.advance_batch_to(watermark_ms),
ShardSlot::Frozen { .. } => Some(None),
})
.collect();
let out = concat(&per);
self.enforce_budget();
out
}
fn flush(&mut self) -> Vec<WindowRow> {
self.shards
.iter_mut()
.filter_map(|s| match s {
ShardSlot::Hot(op) => Some(op.flush()),
ShardSlot::Frozen { .. } => None,
})
.flatten()
.collect()
}
fn flush_batch(&mut self) -> Option<Option<RecordBatch>> {
let per: Vec<_> = self
.shards
.iter_mut()
.map(|s| match s {
ShardSlot::Hot(op) => op.flush_batch(),
ShardSlot::Frozen { .. } => Some(None),
})
.collect();
concat(&per)
}
fn state_bytes(&self) -> usize {
self.resident_bytes()
}
fn open_windows(&self) -> usize {
self.shards
.iter()
.map(|s| match s {
ShardSlot::Hot(op) => op.open_windows(),
ShardSlot::Frozen { .. } => 0,
})
.sum()
}
fn snapshot(&mut self) -> Vec<u8> {
let mut out = Vec::new();
out.extend_from_slice(&(self.shards.len() as u32).to_le_bytes());
for s in &mut self.shards {
let snap = match s {
ShardSlot::Hot(op) => op.snapshot(),
ShardSlot::Frozen { path, .. } => {
std::fs::read(path).unwrap_or_else(|e| panic!("read frozen shard: {e}"))
}
};
out.extend_from_slice(&(snap.len() as u32).to_le_bytes());
out.extend_from_slice(&snap);
}
out
}
fn load_snapshot(&mut self, bytes: &[u8]) -> Result<(), String> {
let mut pos = 0usize;
let read_u32 = |p: &mut usize| -> Result<usize, String> {
let v = bytes
.get(*p..*p + 4)
.ok_or("sharded snapshot: truncated")?
.try_into()
.map_err(|_| "sharded snapshot: len")?;
*p += 4;
Ok(u32::from_le_bytes(v) as usize)
};
let k = read_u32(&mut pos)?;
if k != self.shards.len() {
return Err(format!(
"sharded snapshot: {k} shards != this operator's {}",
self.shards.len()
));
}
for i in 0..k {
let len = read_u32(&mut pos)?;
let frame = bytes.get(pos..pos + len).ok_or("sharded snapshot: truncated frame")?;
pos += len;
let mut op = (self.mk)();
op.load_snapshot(frame)?;
self.shards[i] = ShardSlot::Hot(op);
self.touch[i] = 0;
}
self.enforce_budget();
Ok(())
}
fn shard_files_supported(&self) -> bool {
true
}
fn snapshot_shard_files(&mut self, dir: &Path, tag: &str) -> Vec<(usize, PathBuf)> {
use std::io::Write;
let mut out = Vec::new();
let hot_path = dir.join(format!("{tag}-hot.arrow"));
{
let mut f = std::io::BufWriter::new(
std::fs::File::create(&hot_path).unwrap_or_else(|e| panic!("checkpoint file {hot_path:?}: {e}")),
);
let hot: Vec<usize> = (0..self.shards.len())
.filter(|&i| matches!(self.shards[i], ShardSlot::Hot(_)))
.collect();
f.write_all(&(hot.len() as u32).to_le_bytes()).expect("write hot count");
for i in hot {
let snap = match &mut self.shards[i] {
ShardSlot::Hot(op) => op.snapshot(),
ShardSlot::Frozen { .. } => unreachable!("filtered to hot"),
};
f.write_all(&(i as u32).to_le_bytes()).expect("write shard idx");
f.write_all(&(snap.len() as u32).to_le_bytes()).expect("write shard len");
f.write_all(&snap).expect("write shard");
}
f.flush().expect("flush checkpoint file");
}
out.push((HOT_COMBINED, hot_path));
for i in 0..self.shards.len() {
if let ShardSlot::Frozen { path: frozen } = &self.shards[i] {
let link = dir.join(format!("{tag}-frz{i}.arrow"));
if std::fs::hard_link(frozen, &link).is_err() {
std::fs::copy(frozen, &link).unwrap_or_else(|e| panic!("stage frozen shard {i}: {e}"));
}
out.push((i, link));
}
}
out
}
fn load_shard_files(&mut self, files: &[(usize, PathBuf)]) -> Result<(), String> {
let hot_path = files
.iter()
.find(|(idx, _)| *idx == HOT_COMBINED)
.map(|(_, p)| p.clone())
.ok_or("sharded restore: no hot file")?;
for (idx, path) in files {
if *idx != HOT_COMBINED {
if *idx >= self.shards.len() {
return Err(format!("sharded restore: frozen shard {idx} out of {}", self.shards.len()));
}
self.shards[*idx] = ShardSlot::Frozen { path: path.clone() };
self.touch[*idx] = 0;
}
}
let bytes = std::fs::read(&hot_path).map_err(|e| format!("read hot file: {e}"))?;
let mut pos = 0usize;
let read_u32 = |b: &[u8], p: &mut usize| -> Result<usize, String> {
let v = b.get(*p..*p + 4).ok_or("hot file: truncated")?;
*p += 4;
Ok(u32::from_le_bytes(v.try_into().unwrap()) as usize)
};
let n_hot = read_u32(&bytes, &mut pos)?;
let mut idxs: Vec<usize> = Vec::with_capacity(n_hot);
let mut frames: Vec<&[u8]> = Vec::with_capacity(n_hot);
for _ in 0..n_hot {
let idx = read_u32(&bytes, &mut pos)?;
let len = read_u32(&bytes, &mut pos)?;
let fr = bytes.get(pos..pos + len).ok_or("hot file: truncated frame")?;
pos += len;
if idx >= self.shards.len() {
return Err(format!("hot file: shard {idx} out of {}", self.shards.len()));
}
idxs.push(idx);
frames.push(fr);
}
let mut ops: Vec<Box<dyn WindowOperator + Send>> = (0..n_hot).map(|_| (self.mk)()).collect();
let ncpu = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).max(1);
let chunk = n_hot.div_ceil(ncpu).max(1);
let mut first_err: Option<String> = None;
std::thread::scope(|s| {
let mut handles = Vec::new();
for (ops_chunk, frames_chunk) in ops.chunks_mut(chunk).zip(frames.chunks(chunk)) {
handles.push(s.spawn(move || -> Result<(), String> {
for (op, fr) in ops_chunk.iter_mut().zip(frames_chunk) {
op.load_snapshot(fr)?;
}
Ok(())
}));
}
for h in handles {
if let Err(e) = h.join().expect("shard restore thread panicked") {
first_err.get_or_insert(e);
}
}
});
if let Some(e) = first_err {
return Err(e);
}
for (idx, op) in idxs.into_iter().zip(ops) {
self.shards[idx] = ShardSlot::Hot(op);
self.touch[idx] = 0;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::batch_window::WindowAggBatch;
use crate::Agg;
use arrow::array::{Float64Array, Int64Array, StringArray};
use arrow::datatypes::{Field, Schema};
use std::sync::Arc;
pub(super) fn batch(rows: &[(&str, f64, i64)]) -> (RecordBatch, Vec<i64>) {
let schema = Arc::new(Schema::new(vec![
Field::new("k", DataType::Utf8, true),
Field::new("v", DataType::Float64, true),
Field::new("ts", DataType::Int64, true),
]));
let k: StringArray = rows.iter().map(|r| r.0).collect::<Vec<_>>().into();
let v: Float64Array = rows.iter().map(|r| r.1).collect::<Vec<_>>().into();
let ts: Int64Array = rows.iter().map(|r| r.2).collect::<Vec<_>>().into();
let b = RecordBatch::try_new(schema, vec![Arc::new(k), Arc::new(v), Arc::new(ts)]).unwrap();
let t = rows.iter().map(|r| r.2).collect();
(b, t)
}
pub(super) fn rows_of(b: &RecordBatch) -> Vec<String> {
use arrow::util::display::{ArrayFormatter, FormatOptions};
let opts = FormatOptions::default();
let fmts: Vec<_> = b
.columns()
.iter()
.map(|c| ArrayFormatter::try_new(c, &opts).unwrap())
.collect();
let mut out: Vec<String> = (0..b.num_rows())
.map(|i| {
fmts.iter()
.map(|f| f.value(i).to_string())
.collect::<Vec<_>>()
.join("|")
})
.collect();
out.sort();
out
}
pub(super) fn drain(op: &mut dyn WindowOperator, wm: i64) -> Vec<String> {
match op.advance_batch_to(wm) {
Some(Some(b)) => rows_of(&b),
_ => Vec::new(),
}
}
#[test]
fn a_sharded_window_matches_one_unsharded_operator_row_for_row() {
let aggs = || vec![Agg::count("n"), Agg::of(crate::AggOp::Sum, "v", "s")];
let mut plain = WindowAggBatch::tumbling(1000, 0, vec!["k".into()], aggs());
let mut sharded = ShardedWindow::new(
4,
move || Box::new(WindowAggBatch::tumbling(1000, 0, vec!["k".into()], aggs())),
vec!["k".into()],
);
plain.expect_partitions(&[0], 0);
sharded.expect_partitions(&[0], 0);
let mut data = Vec::new();
for w in 0..5i64 {
for key in 0..40i64 {
data.push((format!("k{key}"), (key + w) as f64, w * 1000 + key % 900));
}
}
for chunk in data.chunks(37) {
let rows: Vec<(&str, f64, i64)> = chunk.iter().map(|(k, v, t)| (k.as_str(), *v, *t)).collect();
let (b, t) = batch(&rows);
assert_eq!(plain.push_batch(0, &b, &t), sharded.push_batch(0, &b, &t));
}
for wm in [1500i64, 3500, 5500] {
assert_eq!(drain(&mut plain, wm), drain(&mut sharded, wm), "mismatch at wm {wm}");
}
let snap = sharded.snapshot();
let mut restored = ShardedWindow::new(
4,
|| {
Box::new(WindowAggBatch::tumbling(
1000,
0,
vec!["k".into()],
vec![Agg::count("n"), Agg::of(crate::AggOp::Sum, "v", "s")],
))
},
vec!["k".into()],
);
restored.load_snapshot(&snap).unwrap();
assert_eq!(drain(&mut sharded, 100_000), drain(&mut restored, 100_000));
}
#[test]
fn a_sharded_updating_window_early_fires_through_every_shard_and_closes_exactly() {
use crate::Trigger;
let aggs = || vec![Agg::count("n"), Agg::of(crate::AggOp::Sum, "v", "s")];
let mut updating = ShardedWindow::new(
4,
move || Box::new(WindowAggBatch::tumbling(1000, 0, vec!["k".into()], aggs()).with_trigger(Trigger::EveryRows(2))),
vec!["k".into()],
);
let mut plain = ShardedWindow::new(
4,
move || Box::new(WindowAggBatch::tumbling(1000, 0, vec!["k".into()], aggs())),
vec!["k".into()],
);
updating.expect_partitions(&[0], 0);
plain.expect_partitions(&[0], 0);
let (b, t) = batch(&[
("a", 1.0, 100),
("b", 2.0, 200),
("c", 3.0, 300),
("a", 4.0, 400),
("b", 5.0, 500),
]);
updating.push_batch(0, &b, &t);
plain.push_batch(0, &b, &t);
assert!(!drain(&mut updating, 500).is_empty(), "trigger early-fired through the shards");
assert!(drain(&mut plain, 500).is_empty(), "no trigger → no early emission");
assert_eq!(drain(&mut updating, 1000), drain(&mut plain, 1000));
}
#[test]
fn shard_files_checkpoint_restores_frozen_shards_lazily_and_exactly() {
fn mk6() -> Box<dyn WindowOperator + Send> {
Box::new(WindowAggBatch::tumbling(
1000,
0,
vec!["k".into()],
vec![Agg::count("n"), Agg::of(crate::AggOp::Sum, "v", "s")],
))
}
let build = || ShardedWindow::new(6, mk6, vec!["k".into()]);
let mut whole = build();
let evdir = tempfile::tempdir().unwrap();
let mut byfile = build().with_eviction(evdir.path().to_path_buf(), 1);
whole.expect_partitions(&[0], 0);
byfile.expect_partitions(&[0], 0);
for w in 0..3i64 {
let rows: Vec<(String, f64, i64)> =
(0..30i64).map(|key| (format!("k{key}"), (key + w) as f64, w * 1000 + 10)).collect();
let r: Vec<(&str, f64, i64)> = rows.iter().map(|(k, v, t)| (k.as_str(), *v, *t)).collect();
let (b, t) = batch(&r);
assert_eq!(whole.push_batch(0, &b, &t), byfile.push_batch(0, &b, &t));
let wm = (w + 1) * 1000 + 5;
assert_eq!(drain(&mut whole, wm), drain(&mut byfile, wm), "mismatch firing window {w}");
}
let ckpt = tempfile::tempdir().unwrap();
let files = byfile.snapshot_shard_files(ckpt.path(), "s0-e7");
assert_eq!(files.iter().filter(|(i, _)| *i == HOT_COMBINED).count(), 1, "one combined hot file");
let n_frozen = files.iter().filter(|(i, _)| *i != HOT_COMBINED).count();
assert!(n_frozen >= 1, "fully-fired windows should leave window-less shards to freeze (got {n_frozen})");
let mut restored = build();
restored.expect_partitions(&[0], 0);
restored.load_shard_files(&files).unwrap();
assert!(
restored.resident_shards() < 6,
"frozen shards must restore lazily (resident {})",
restored.resident_shards()
);
let (lb, lt) = batch(&[("k5", 9.0, 5000), ("k17", 8.0, 5000)]);
assert_eq!(whole.push_batch(0, &lb, <), restored.push_batch(0, &lb, <));
assert_eq!(drain(&mut whole, 100_000), drain(&mut restored, 100_000));
}
}
#[cfg(test)]
mod eviction_tests {
use super::tests::*;
use super::*;
use crate::batch_window::WindowAggBatch;
use crate::{Agg, AggOp};
#[test]
fn eviction_stays_exact_and_bounds_resident_shards() {
let mk = || vec![Agg::count("n"), Agg::of(AggOp::Sum, "v", "s")];
let mut plain = WindowAggBatch::tumbling(1000, 0, vec!["k".into()], mk());
let dir = tempfile::tempdir().unwrap();
let mut sharded = ShardedWindow::new(
8,
move || Box::new(WindowAggBatch::tumbling(1000, 0, vec!["k".into()], mk())),
vec!["k".into()],
)
.with_eviction(dir.path().to_path_buf(), 1);
plain.expect_partitions(&[0], 0);
sharded.expect_partitions(&[0], 0);
let mut min_resident = 8usize;
for w in 0..6i64 {
let rows: Vec<(String, f64, i64)> = (0..60i64)
.map(|key| (format!("k{key}"), (key + w) as f64, w * 1000 + 10))
.collect();
let r: Vec<(&str, f64, i64)> = rows.iter().map(|(k, v, t)| (k.as_str(), *v, *t)).collect();
let (b, t) = batch(&r);
assert_eq!(plain.push_batch(0, &b, &t), sharded.push_batch(0, &b, &t));
let wm = (w + 1) * 1000 + 5;
assert_eq!(
drain(&mut plain, wm),
drain(&mut sharded, wm),
"mismatch firing window {w}"
);
min_resident = min_resident.min(sharded.resident_shards());
}
assert!(
min_resident < 8,
"eviction never froze a shard (resident stayed {min_resident})"
);
let (b, t) = batch(&[("k7", 1.0, 6100), ("k42", 2.0, 6100)]);
assert_eq!(plain.push_batch(0, &b, &t), sharded.push_batch(0, &b, &t));
assert_eq!(drain(&mut plain, 100_000), drain(&mut sharded, 100_000));
}
}