use rdkafka::config::ClientConfig;
use rdkafka::producer::{BaseProducer, BaseRecord, Producer};
use serde_json::{json, Value as J};
use std::time::Duration;
#[derive(Clone)]
pub struct NexmarkCfg {
pub dataset: String,
pub topic: String,
pub brokers: String,
pub events: u64,
pub seed: u64,
pub step_ms: i64,
pub rate: u64,
pub partitions: i32,
pub delay_ms: u64,
pub tail: bool,
pub tail_delay_ms: u64,
pub inline: bool,
}
pub fn parse(dataset: &str, ds: &J, default_brokers: &str) -> Option<Result<NexmarkCfg, String>> {
if ds["connector"].as_str() != Some("nexmark") {
return None;
}
let inline = ds["inline"].as_bool().unwrap_or(false);
let topic = match ds["topic"].as_str() {
Some(t) => t,
None if inline => "",
None => {
return Some(Err(format!(
"pipeline.toml: [datasets.{dataset}] connector=\"nexmark\" needs `topic = \"…\"` (or `inline = true`)"
)))
}
};
Some(Ok(NexmarkCfg {
dataset: dataset.to_string(),
topic: topic.to_string(),
brokers: ds["brokers"].as_str().unwrap_or(default_brokers).to_string(),
events: ds["events"].as_u64().unwrap_or(100_000),
seed: ds["seed"].as_u64().unwrap_or(42),
step_ms: ds["stepMs"].as_i64().unwrap_or(10).max(0),
rate: ds["rate"].as_u64().unwrap_or(0),
partitions: ds["partitions"].as_i64().unwrap_or(4).max(1) as i32,
delay_ms: ds["delayMs"].as_u64().unwrap_or(0),
tail: ds["tail"].as_bool().unwrap_or(true),
tail_delay_ms: ds["tailDelayMs"].as_u64().unwrap_or(8_000),
inline,
}))
}
struct Lcg(u64);
impl Lcg {
fn next(&mut self) -> u64 {
self.0 = self
.0
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
self.0 >> 16 }
fn below(&mut self, n: u64) -> u64 {
if n == 0 {
0
} else {
self.next() % n
}
}
}
const US_STATES: [&str; 10] = ["OR", "ID", "CA", "WA", "NV", "AZ", "UT", "MT", "WY", "CO"];
const CITIES: [&str; 10] = [
"Portland",
"Boise",
"Sacramento",
"Seattle",
"Reno",
"Phoenix",
"Provo",
"Helena",
"Cheyenne",
"Denver",
];
const CHANNELS: [&str; 4] = ["Google", "Facebook", "Baidu", "Apple"];
const CATEGORIES: u64 = 5;
const FIRST_ID: u64 = 1000;
const BASE_TIME_MS: i64 = 1_000_000; pub const SENTINEL_ID: u64 = 1;
const _: () = assert!(SENTINEL_ID < FIRST_ID);
pub const SENTINEL_AHEAD_MS: i64 = 10_000_000;
fn event(n: u64, seed: u64, step_ms: i64, persons: u64, auctions: u64) -> (String, J, char) {
let mut r = Lcg(seed ^ (n.wrapping_mul(0x9E3779B97F4A7C15)));
r.next();
let date_time = BASE_TIME_MS + (n as i64) * step_ms;
match n % 50 {
0 => {
let id = FIRST_ID + persons;
let key = format!("p{id}");
let payload = json!({
"type": "person",
"id": id,
"name": format!("person-{id}"),
"emailAddress": format!("person-{id}@example.com"),
"creditCard": format!("{:04} {:04} {:04} {:04}", r.below(10000), r.below(10000), r.below(10000), r.below(10000)),
"city": CITIES[r.below(CITIES.len() as u64) as usize],
"state": US_STATES[r.below(US_STATES.len() as u64) as usize],
"dateTime": date_time,
});
(key, payload, 'p')
}
1..=3 => {
let id = FIRST_ID + auctions;
let key = format!("a{id}");
let initial = 1 + r.below(1000);
let seller = FIRST_ID + r.below(persons.max(1));
let payload = json!({
"type": "auction",
"id": id,
"itemName": format!("item-{id}"),
"initialBid": initial,
"reserve": initial + r.below(1000),
"dateTime": date_time,
"expires": date_time + 10_000 + r.below(100_000) as i64,
"seller": seller,
"category": r.below(CATEGORIES),
});
(key, payload, 'a')
}
_ => {
let auction = if r.below(2) == 0 {
FIRST_ID + auctions.saturating_sub(1)
} else {
FIRST_ID + r.below(auctions.max(1))
};
let bidder = FIRST_ID + r.below(persons.max(1));
let key = format!("a{auction}");
let price = 1 + r.below(10_000);
let hot_channel = CHANNELS[r.below(CHANNELS.len() as u64) as usize];
let (channel, url) = bid_channel_url(hot_channel, auction, bidder, date_time);
let payload = json!({
"type": "bid",
"auction": auction,
"bidder": bidder,
"price": price,
"channel": channel,
"url": url,
"dateTime": date_time,
});
(key, payload, 'b')
}
}
}
pub fn spawn(cfg: NexmarkCfg) -> std::thread::JoinHandle<()> {
std::thread::spawn(move || {
if cfg.delay_ms > 0 {
std::thread::sleep(Duration::from_millis(cfg.delay_ms));
}
let producer: BaseProducer = match ClientConfig::new()
.set("bootstrap.servers", &cfg.brokers)
.set("enable.idempotence", "true")
.set("queue.buffering.max.messages", "1000000")
.create()
{
Ok(p) => p,
Err(e) => {
eprintln!("nexmark[{}]: producer create failed: {e}", cfg.dataset);
return;
}
};
let started = std::time::Instant::now();
let (mut persons, mut auctions) = (0u64, 0u64);
for n in 0..cfg.events {
let (key, payload, kind) = event(n, cfg.seed, cfg.step_ms, persons, auctions);
match kind {
'p' => persons += 1,
'a' => auctions += 1,
_ => {}
}
let bytes = serde_json::to_vec(&payload).expect("nexmark event serializes");
let mut rec = BaseRecord::to(&cfg.topic).key(&key).payload(&bytes);
loop {
match producer.send(rec) {
Ok(()) => break,
Err((_, r)) => {
producer.poll(Duration::from_millis(2));
rec = r;
}
}
}
producer.poll(Duration::ZERO);
if cfg.rate > 0 && n % 500 == 0 {
let due = started.elapsed().as_secs_f64() * cfg.rate as f64;
if (n as f64) > due {
std::thread::sleep(Duration::from_millis(
(((n as f64 - due) / cfg.rate as f64) * 1000.0) as u64 + 1,
));
}
}
}
let _ = producer.flush(Duration::from_secs(60));
println!(
"nexmark[{}]: produced {} real events in {:.1}s",
cfg.dataset,
cfg.events,
started.elapsed().as_secs_f64()
);
if cfg.tail {
let _ = producer.flush(Duration::from_secs(60));
std::thread::sleep(Duration::from_millis(cfg.tail_delay_ms));
let t = BASE_TIME_MS + (cfg.events as i64) * cfg.step_ms + SENTINEL_AHEAD_MS;
let sentinels = [
json!({"type": "person", "id": SENTINEL_ID, "name": "sentinel", "emailAddress": "s@s", "creditCard": "0", "city": "Nowhere", "state": "XX", "dateTime": t}),
json!({"type": "auction", "id": SENTINEL_ID, "itemName": "sentinel", "initialBid": 0, "reserve": 0, "dateTime": t, "expires": t, "seller": SENTINEL_ID, "category": 0}),
json!({"type": "bid", "auction": SENTINEL_ID, "bidder": SENTINEL_ID, "price": 0, "channel": "sentinel", "dateTime": t}),
];
for p in 0..cfg.partitions {
for payload in &sentinels {
let bytes = serde_json::to_vec(payload).expect("sentinel serializes");
let mut rec = BaseRecord::to(&cfg.topic).key("sentinel").payload(&bytes).partition(p);
loop {
match producer.send(rec) {
Ok(()) => break,
Err((_, r)) => {
producer.poll(Duration::from_millis(2));
rec = r;
}
}
}
}
}
}
let _ = producer.flush(Duration::from_secs(60));
println!(
"nexmark[{}]: generated {} events → {} ({} persons, {} auctions, {} bids) in {:.1}s",
cfg.dataset,
cfg.events,
cfg.topic,
persons,
auctions,
cfg.events - persons - auctions,
started.elapsed().as_secs_f64()
);
})
}
fn bid_channel_url(hot_channel: &str, auction: u64, bidder: u64, date_time: i64) -> (String, String) {
let mut r = Lcg(auction.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ bidder.rotate_left(17) ^ (date_time as u64));
let seg = |r: &mut Lcg| -> String {
(0..4)
.map(|_| {
let v = r.below(36) as u8;
(if v < 10 { b'0' + v } else { b'a' + v - 10 }) as char
})
.collect()
};
let (a, b, c) = (seg(&mut r), seg(&mut r), seg(&mut r));
if r.below(10) == 0 {
let id = r.below(1_000);
(
format!("channel-{id}"),
format!("https://www.nexmark.com/{a}/{b}/{c}/item.htm?query=1&channel_id={id}"),
)
} else {
(
hot_channel.to_string(),
format!("https://www.nexmark.com/{a}/{b}/{c}/item.htm?query=1"),
)
}
}
pub struct NexmarkInline {
pub cfg: NexmarkCfg,
}
impl fv_streams_engine::InlineSource for NexmarkInline {
fn name(&self) -> String {
format!("nexmark:{}", self.cfg.dataset)
}
fn tasks(&self) -> usize {
self.cfg.partitions.max(1) as usize
}
fn open(
&self,
task: usize,
tasks: usize,
start: Option<i64>,
watermark_column: Option<&str>,
) -> Result<Box<dyn fv_streams_engine::dataflow::Source + Send>, String> {
let events = self.cfg.events;
let (next, sentinels_left) = match start.map(|s| s.max(0) as u64) {
Some(s) if s > events => (events, 3 - (s - events).min(3) as u32),
Some(s) => (s, 3),
None => (0, 3),
};
Ok(Box::new(NexmarkSource {
cfg: self.cfg.clone(),
task,
tasks,
next,
persons: 0,
auctions: 0,
counted_through: 0,
stamps: watermark_column == Some("dateTime"),
batch_max: std::env::var("STREAM_BATCH_MAX")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(5000),
started: std::time::Instant::now(),
reported_real: false,
sentinels_left,
done: false,
}))
}
}
struct NexmarkSource {
cfg: NexmarkCfg,
task: usize,
tasks: usize,
next: u64,
persons: u64,
auctions: u64,
counted_through: u64,
stamps: bool,
batch_max: usize,
started: std::time::Instant,
reported_real: bool,
sentinels_left: u32,
done: bool,
}
impl NexmarkSource {
fn record(&self) -> Vec<u8> {
let pos = if self.next >= self.cfg.events {
self.cfg.events + (3 - self.sentinels_left) as u64
} else {
self.next
};
fv_streams_engine::dataflow::position_record(
&format!("nexmark:{}", self.cfg.dataset),
self.task as i32,
pos as i64,
)
}
fn count_through(&mut self, n: u64) {
while self.counted_through < n {
let i = self.counted_through;
match i % 50 {
0 => self.persons += 1,
1..=3 => self.auctions += 1,
_ => {}
}
self.counted_through = i + 1;
}
}
}
fn nexmark_schema() -> arrow::datatypes::SchemaRef {
use arrow::datatypes::{DataType, Field, Schema};
std::sync::Arc::new(Schema::new(vec![
Field::new("type", DataType::Utf8, true),
Field::new("id", DataType::Int64, true),
Field::new("name", DataType::Utf8, true),
Field::new("emailAddress", DataType::Utf8, true),
Field::new("creditCard", DataType::Utf8, true),
Field::new("city", DataType::Utf8, true),
Field::new("state", DataType::Utf8, true),
Field::new("dateTime", DataType::Int64, true),
Field::new("itemName", DataType::Utf8, true),
Field::new("initialBid", DataType::Int64, true),
Field::new("reserve", DataType::Int64, true),
Field::new("expires", DataType::Int64, true),
Field::new("seller", DataType::Int64, true),
Field::new("category", DataType::Int64, true),
Field::new("auction", DataType::Int64, true),
Field::new("bidder", DataType::Int64, true),
Field::new("price", DataType::Int64, true),
Field::new("channel", DataType::Utf8, true),
Field::new("url", DataType::Utf8, true),
]))
}
struct EventColumns {
kind: arrow::array::StringBuilder,
id: arrow::array::Int64Builder,
name: arrow::array::StringBuilder,
email: arrow::array::StringBuilder,
credit_card: arrow::array::StringBuilder,
city: arrow::array::StringBuilder,
state: arrow::array::StringBuilder,
date_time: arrow::array::Int64Builder,
item_name: arrow::array::StringBuilder,
initial_bid: arrow::array::Int64Builder,
reserve: arrow::array::Int64Builder,
expires: arrow::array::Int64Builder,
seller: arrow::array::Int64Builder,
category: arrow::array::Int64Builder,
auction: arrow::array::Int64Builder,
bidder: arrow::array::Int64Builder,
price: arrow::array::Int64Builder,
channel: arrow::array::StringBuilder,
url: arrow::array::StringBuilder,
keys: Vec<Option<String>>,
max_time: i64,
}
impl EventColumns {
fn with_capacity(n: usize) -> Self {
use arrow::array::{Int64Builder, StringBuilder};
EventColumns {
kind: StringBuilder::with_capacity(n, n * 4),
id: Int64Builder::with_capacity(n),
name: StringBuilder::with_capacity(n, n * 4),
email: StringBuilder::with_capacity(n, n * 4),
credit_card: StringBuilder::with_capacity(n, n * 4),
city: StringBuilder::with_capacity(n, n * 4),
state: StringBuilder::with_capacity(n, n * 4),
date_time: Int64Builder::with_capacity(n),
item_name: StringBuilder::with_capacity(n, n * 4),
initial_bid: Int64Builder::with_capacity(n),
reserve: Int64Builder::with_capacity(n),
expires: Int64Builder::with_capacity(n),
seller: Int64Builder::with_capacity(n),
category: Int64Builder::with_capacity(n),
auction: Int64Builder::with_capacity(n),
bidder: Int64Builder::with_capacity(n),
price: Int64Builder::with_capacity(n),
channel: StringBuilder::with_capacity(n, n * 8),
url: StringBuilder::with_capacity(n, n * 48),
keys: Vec::with_capacity(n),
max_time: i64::MIN,
}
}
fn len(&self) -> usize {
self.keys.len()
}
fn push(&mut self, n: u64, seed: u64, step_ms: i64, persons: u64, auctions: u64) {
let mut r = Lcg(seed ^ (n.wrapping_mul(0x9E3779B97F4A7C15)));
r.next();
let date_time = BASE_TIME_MS + (n as i64) * step_ms;
self.max_time = self.max_time.max(date_time);
self.date_time.append_value(date_time);
match n % 50 {
0 => {
let id = FIRST_ID + persons;
self.keys.push(Some(format!("p{id}")));
self.kind.append_value("person");
self.id.append_value(id as i64);
self.name.append_value(format!("person-{id}"));
self.email.append_value(format!("person-{id}@example.com"));
self.credit_card.append_value(format!(
"{:04} {:04} {:04} {:04}",
r.below(10000),
r.below(10000),
r.below(10000),
r.below(10000)
));
self.city.append_value(CITIES[r.below(CITIES.len() as u64) as usize]);
self.state
.append_value(US_STATES[r.below(US_STATES.len() as u64) as usize]);
for b in [&mut self.item_name, &mut self.channel, &mut self.url] {
b.append_null();
}
for b in [
&mut self.initial_bid,
&mut self.reserve,
&mut self.expires,
&mut self.seller,
&mut self.category,
&mut self.auction,
&mut self.bidder,
&mut self.price,
] {
b.append_null();
}
}
1..=3 => {
let id = FIRST_ID + auctions;
self.keys.push(Some(format!("a{id}")));
let initial = 1 + r.below(1000);
let seller = FIRST_ID + r.below(persons.max(1));
self.kind.append_value("auction");
self.id.append_value(id as i64);
self.item_name.append_value(format!("item-{id}"));
self.initial_bid.append_value(initial as i64);
self.reserve.append_value((initial + r.below(1000)) as i64);
self.expires.append_value(date_time + 10_000 + r.below(100_000) as i64);
self.seller.append_value(seller as i64);
self.category.append_value(r.below(CATEGORIES) as i64);
for b in [
&mut self.name,
&mut self.email,
&mut self.credit_card,
&mut self.city,
&mut self.state,
&mut self.channel,
&mut self.url,
] {
b.append_null();
}
for b in [&mut self.auction, &mut self.bidder, &mut self.price] {
b.append_null();
}
}
_ => {
let auction = if r.below(2) == 0 {
FIRST_ID + auctions.saturating_sub(1)
} else {
FIRST_ID + r.below(auctions.max(1))
};
let bidder = FIRST_ID + r.below(persons.max(1));
self.keys.push(Some(format!("a{auction}")));
let price = 1 + r.below(10_000);
let hot_channel = CHANNELS[r.below(CHANNELS.len() as u64) as usize];
let (channel, url) = bid_channel_url(hot_channel, auction, bidder, date_time);
self.kind.append_value("bid");
self.auction.append_value(auction as i64);
self.bidder.append_value(bidder as i64);
self.price.append_value(price as i64);
self.channel.append_value(channel);
self.url.append_value(url);
for b in [
&mut self.name,
&mut self.email,
&mut self.credit_card,
&mut self.city,
&mut self.state,
&mut self.item_name,
] {
b.append_null();
}
for b in [
&mut self.id,
&mut self.initial_bid,
&mut self.reserve,
&mut self.expires,
&mut self.seller,
&mut self.category,
] {
b.append_null();
}
}
}
}
fn push_sentinel(&mut self, which: u32, t: i64) {
use arrow::array::ArrayBuilder;
let row = (self.keys.len(), which);
self.max_time = self.max_time.max(t);
self.date_time.append_value(t);
match row.1 {
3 => {
self.keys.push(Some("p1".into()));
self.kind.append_value("person");
self.id.append_value(SENTINEL_ID as i64);
self.name.append_value("sentinel");
self.email.append_value("s@s");
self.credit_card.append_value("0");
self.city.append_value("Nowhere");
self.state.append_value("XX");
}
2 => {
self.keys.push(Some("a1".into()));
self.kind.append_value("auction");
self.id.append_value(SENTINEL_ID as i64);
self.item_name.append_value("sentinel");
self.initial_bid.append_value(0);
self.reserve.append_value(0);
self.expires.append_value(t);
self.seller.append_value(SENTINEL_ID as i64);
self.category.append_value(0);
}
_ => {
self.keys.push(Some("a1".into()));
self.kind.append_value("bid");
self.auction.append_value(SENTINEL_ID as i64);
self.bidder.append_value(SENTINEL_ID as i64);
self.price.append_value(0);
self.channel.append_value("sentinel");
self.url.append_value("");
}
}
let n = self.keys.len();
macro_rules! pad {
($($b:expr),*) => { $( while $b.len() < n { $b.append_null(); } )* };
}
pad!(
self.kind,
self.id,
self.name,
self.email,
self.credit_card,
self.city,
self.state,
self.item_name,
self.initial_bid,
self.reserve,
self.expires,
self.seller,
self.category,
self.auction,
self.bidder,
self.price,
self.channel,
self.url
);
}
fn finish(mut self, task: usize) -> arrow::array::RecordBatch {
use arrow::array::ArrayRef;
use std::sync::Arc;
let cols: Vec<ArrayRef> = vec![
Arc::new(self.kind.finish()),
Arc::new(self.id.finish()),
Arc::new(self.name.finish()),
Arc::new(self.email.finish()),
Arc::new(self.credit_card.finish()),
Arc::new(self.city.finish()),
Arc::new(self.state.finish()),
Arc::new(self.date_time.finish()),
Arc::new(self.item_name.finish()),
Arc::new(self.initial_bid.finish()),
Arc::new(self.reserve.finish()),
Arc::new(self.expires.finish()),
Arc::new(self.seller.finish()),
Arc::new(self.category.finish()),
Arc::new(self.auction.finish()),
Arc::new(self.bidder.finish()),
Arc::new(self.price.finish()),
Arc::new(self.channel.finish()),
Arc::new(self.url.finish()),
];
let batch = arrow::array::RecordBatch::try_new(nexmark_schema(), cols).expect("nexmark batch");
fv_streams_engine::decode::with_partition(
&fv_streams_engine::decode::with_keys(&batch, &self.keys),
task as i32,
)
}
}
impl fv_streams_engine::dataflow::Source for NexmarkSource {
fn poll(&mut self, out: &mut fv_streams_engine::dataflow::Out) -> fv_streams_engine::dataflow::Poll {
use fv_streams_engine::dataflow::Poll;
if self.done {
return Poll::Done;
}
let mut cols = EventColumns::with_capacity(self.batch_max);
while cols.len() < self.batch_max && self.next < self.cfg.events {
let n = self.next;
self.next += 1;
if n as usize % self.tasks != self.task {
continue;
}
self.count_through(n);
cols.push(n, self.cfg.seed, self.cfg.step_ms, self.persons, self.auctions);
if self.cfg.rate > 0 && n.is_multiple_of(500) {
let due = self.started.elapsed().as_secs_f64() * self.cfg.rate as f64;
if (n as f64) > due {
std::thread::sleep(Duration::from_millis(
(((n as f64 - due) / self.cfg.rate as f64) * 1000.0) as u64 + 1,
));
}
}
}
if self.next >= self.cfg.events {
if !self.reported_real {
self.reported_real = true;
if self.task == 0 {
println!(
"nexmark[{}]: produced {} real events in {:.1}s",
self.cfg.dataset,
self.cfg.events,
self.started.elapsed().as_secs_f64()
);
}
}
if self.sentinels_left > 0 && cols.len() == 0 {
let t = BASE_TIME_MS + (self.cfg.events as i64) * self.cfg.step_ms + SENTINEL_AHEAD_MS;
cols.push_sentinel(self.sentinels_left, t);
self.sentinels_left -= 1;
}
}
if cols.len() == 0 {
if self.next >= self.cfg.events && self.sentinels_left == 0 {
self.done = true;
if self.task == 0 {
println!(
"nexmark[{}]: generated {} events in-process ({} task(s)) in {:.1}s",
self.cfg.dataset,
self.cfg.events,
self.tasks,
self.started.elapsed().as_secs_f64()
);
}
return Poll::Done;
}
return Poll::Idle;
}
let max_time = cols.max_time;
out.push(cols.finish(self.task));
if self.stamps && max_time != i64::MIN {
out.watermark = Some(Some(max_time));
}
Poll::Again
}
fn on_barrier(&mut self, _epoch: u64) -> Vec<u8> {
self.record()
}
fn position(&mut self) -> Vec<u8> {
self.record()
}
}
#[cfg(test)]
mod tests {
#[test]
fn a_drained_position_counts_the_sentinels_and_a_restore_there_emits_nothing_twice() {
use fv_streams_engine::dataflow::{Out, Poll};
use fv_streams_engine::InlineSource;
let cfg = super::NexmarkCfg {
dataset: "events".into(),
topic: String::new(),
brokers: String::new(),
events: 10,
seed: 1,
step_ms: 10,
partitions: 1,
rate: 0,
delay_ms: 0,
tail: true,
tail_delay_ms: 0,
inline: true,
};
let inline = super::NexmarkInline { cfg };
let drain = |start: Option<i64>| -> (usize, Vec<u8>) {
let mut src = inline.open(0, 1, start, None).unwrap();
let mut rows = 0;
loop {
let mut out = Out::default();
match src.poll(&mut out) {
Poll::Again | Poll::Idle => rows += out.batches.iter().map(|b| b.num_rows()).sum::<usize>(),
Poll::Done => break (rows, src.position()),
Poll::Failed(e) => panic!("{e}"),
}
}
};
let (rows, pos) = drain(None);
assert_eq!(rows, 13, "10 events + 3 sentinels");
let v: serde_json::Value = serde_json::from_slice(&pos).unwrap();
assert_eq!(
v["topic"], "nexmark:events",
"the record carries the name the restore looks up"
);
assert_eq!(v["in"]["0"], 13, "the position counts the sentinels");
assert_eq!(
drain(Some(13)).0,
0,
"restored past everything: nothing is emitted again"
);
assert_eq!(
drain(Some(11)).0,
2,
"restored among the sentinels: only the remaining ones"
);
assert_eq!(drain(Some(4)).0, 9, "restored mid-stream: the rest, then the sentinels");
}
#[test]
fn the_columnar_source_matches_the_json_generator_row_for_row() {
use arrow::array::{Array, Int64Array, StringArray};
let (mut persons, mut auctions) = (0u64, 0u64);
let mut cols = EventColumns::with_capacity(300);
let mut expected: Vec<(String, J)> = Vec::new();
for n in 0..300u64 {
let (key, payload, kind) = event(n, 42, 10, persons, auctions);
cols.push(n, 42, 10, persons, auctions);
match kind {
'p' => persons += 1,
'a' => auctions += 1,
_ => {}
}
expected.push((key, payload));
}
let batch = cols.finish(0);
assert_eq!(batch.num_rows(), 300);
let col = |name: &str| batch.column_by_name(name).unwrap().clone();
let kinds = col("type");
let kinds = kinds.as_any().downcast_ref::<StringArray>().unwrap();
let keys = col(fv_streams_engine::decode::META_KEY);
let keys = keys.as_any().downcast_ref::<StringArray>().unwrap();
for (i, (key, payload)) in expected.iter().enumerate() {
assert_eq!(keys.value(i), key, "key of event {i}");
assert_eq!(kinds.value(i), payload["type"].as_str().unwrap());
for f in [
"id",
"dateTime",
"initialBid",
"reserve",
"expires",
"seller",
"category",
"auction",
"bidder",
"price",
] {
let a = col(f);
let a = a.as_any().downcast_ref::<Int64Array>().unwrap();
let want = payload.get(f).and_then(|v| v.as_i64());
let got = if a.is_null(i) { None } else { Some(a.value(i)) };
assert_eq!(got, want, "{f} of event {i}");
}
for f in [
"name",
"emailAddress",
"creditCard",
"city",
"state",
"itemName",
"channel",
"url",
] {
let a = col(f);
let a = a.as_any().downcast_ref::<StringArray>().unwrap();
let want = payload.get(f).and_then(|v| v.as_str());
let got = if a.is_null(i) { None } else { Some(a.value(i)) };
assert_eq!(got, want, "{f} of event {i}");
}
}
}
use super::*;
fn run(n_events: u64, seed: u64) -> Vec<(String, J, char)> {
let (mut persons, mut auctions) = (0u64, 0u64);
(0..n_events)
.map(|n| {
let e = event(n, seed, 10, persons, auctions);
match e.2 {
'p' => persons += 1,
'a' => auctions += 1,
_ => {}
}
e
})
.collect()
}
#[test]
fn deterministic_same_seed_same_stream() {
let a = run(500, 42);
let b = run(500, 42);
assert_eq!(format!("{a:?}"), format!("{b:?}"));
let c = run(500, 43);
assert_ne!(format!("{a:?}"), format!("{c:?}"), "different seed must differ");
}
#[test]
fn proportions_and_referential_integrity() {
let evs = run(5000, 42);
let (mut p, mut a, mut b) = (0u64, 0u64, 0u64);
let mut person_ids = std::collections::HashSet::new();
let mut auction_ids = std::collections::HashSet::new();
for (key, ev, kind) in &evs {
match kind {
'p' => {
p += 1;
person_ids.insert(ev["id"].as_u64().unwrap());
assert!(key.starts_with('p'));
}
'a' => {
a += 1;
auction_ids.insert(ev["id"].as_u64().unwrap());
assert!(
person_ids.contains(&ev["seller"].as_u64().unwrap()),
"auction seller must exist"
);
assert!(key.starts_with('a'));
}
_ => {
b += 1;
assert!(
auction_ids.contains(&ev["auction"].as_u64().unwrap()),
"bid auction must exist"
);
assert!(
person_ids.contains(&ev["bidder"].as_u64().unwrap()),
"bidder must exist"
);
assert_eq!(
*key,
format!("a{}", ev["auction"].as_u64().unwrap()),
"bids keyed by auction"
);
}
}
}
assert_eq!((p, a, b), (5000 / 50, 3 * 5000 / 50, 46 * 5000 / 50));
}
#[test]
fn event_time_is_monotonic_with_step() {
let evs = run(200, 7);
for (i, (_, ev, _)) in evs.iter().enumerate() {
assert_eq!(ev["dateTime"].as_i64().unwrap(), BASE_TIME_MS + (i as i64) * 10);
}
}
#[test]
fn golden_values_pin_the_deterministic_stream() {
let evs = run(52, 42);
let pin = |i: usize, key: &str, kind: char, json: &str| {
assert_eq!(evs[i].0, key, "event {i} key");
assert_eq!(evs[i].2, kind, "event {i} kind");
assert_eq!(serde_json::to_string(&evs[i].1).unwrap(), json, "event {i} payload");
};
pin(
0,
"p1000",
'p',
r#"{"city":"Boise","creditCard":"2172 7803 7043 3278","dateTime":1000000,"emailAddress":"person-1000@example.com","id":1000,"name":"person-1000","state":"AZ","type":"person"}"#,
);
pin(
1,
"a1000",
'a',
r#"{"category":3,"dateTime":1000010,"expires":1064796,"id":1000,"initialBid":166,"itemName":"item-1000","reserve":533,"seller":1000,"type":"auction"}"#,
);
pin(
4,
"a1002",
'b',
r#"{"auction":1002,"bidder":1000,"channel":"Facebook","dateTime":1000040,"price":9312,"type":"bid","url":"https://www.nexmark.com/kma4/6te1/rjvs/item.htm?query=1"}"#,
);
pin(
50,
"p1001",
'p',
r#"{"city":"Sacramento","creditCard":"0905 6287 0028 8090","dateTime":1000500,"emailAddress":"person-1001@example.com","id":1001,"name":"person-1001","state":"NV","type":"person"}"#,
);
}
#[test]
fn pacing_config_defaults_are_safe() {
let cfg = parse("d", &serde_json::json!({"connector": "nexmark", "topic": "t"}), "b")
.unwrap()
.unwrap();
assert_eq!(
(cfg.rate, cfg.delay_ms, cfg.events, cfg.seed, cfg.step_ms),
(0, 0, 100_000, 42, 10)
);
}
#[test]
fn parse_recognizes_only_nexmark_datasets() {
assert!(parse("d", &json!({"topic": "t"}), "b:9092").is_none());
let cfg = parse(
"d",
&json!({"connector": "nexmark", "topic": "t", "events": 5, "seed": 9, "rate": 100}),
"b:9092",
)
.unwrap()
.unwrap();
assert_eq!(
(cfg.events, cfg.seed, cfg.rate, cfg.partitions, cfg.delay_ms),
(5, 9, 100, 4, 0)
);
assert!(cfg.tail, "tail sentinels default ON");
assert_eq!(cfg.tail_delay_ms, 8_000, "quiesce default");
assert!(
parse("d", &json!({"connector": "nexmark"}), "b").unwrap().is_err(),
"topic required"
);
}
}