use rdkafka::config::ClientConfig;
use rdkafka::producer::{BaseProducer, BaseRecord, Producer};
use serde_json::{json, Value as J};
use std::time::Duration;
pub struct NexmarkCfg {
pub dataset: String,
pub topic: String,
pub brokers: String,
pub events: u64,
pub seed: u64,
pub step_ms: i64,
pub rate: u64,
pub partitions: i32,
pub delay_ms: u64,
pub tail: bool,
pub tail_delay_ms: u64,
}
pub fn parse(dataset: &str, ds: &J, default_brokers: &str) -> Option<Result<NexmarkCfg, String>> {
if ds["connector"].as_str() != Some("nexmark") {
return None;
}
let Some(topic) = ds["topic"].as_str() else {
return Some(Err(format!(
"pipeline.toml: [datasets.{dataset}] connector=\"nexmark\" needs `topic = \"…\"`"
)));
};
Some(Ok(NexmarkCfg {
dataset: dataset.to_string(),
topic: topic.to_string(),
brokers: ds["brokers"].as_str().unwrap_or(default_brokers).to_string(),
events: ds["events"].as_u64().unwrap_or(100_000),
seed: ds["seed"].as_u64().unwrap_or(42),
step_ms: ds["stepMs"].as_i64().unwrap_or(10).max(0),
rate: ds["rate"].as_u64().unwrap_or(0),
partitions: ds["partitions"].as_i64().unwrap_or(4).max(1) as i32,
delay_ms: ds["delayMs"].as_u64().unwrap_or(0),
tail: ds["tail"].as_bool().unwrap_or(true),
tail_delay_ms: ds["tailDelayMs"].as_u64().unwrap_or(8_000),
}))
}
struct Lcg(u64);
impl Lcg {
fn next(&mut self) -> u64 {
self.0 = self
.0
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
self.0 >> 16 }
fn below(&mut self, n: u64) -> u64 {
if n == 0 {
0
} else {
self.next() % n
}
}
}
const US_STATES: [&str; 10] = ["OR", "ID", "CA", "WA", "NV", "AZ", "UT", "MT", "WY", "CO"];
const CITIES: [&str; 10] = [
"Portland",
"Boise",
"Sacramento",
"Seattle",
"Reno",
"Phoenix",
"Provo",
"Helena",
"Cheyenne",
"Denver",
];
const CHANNELS: [&str; 4] = ["Google", "Facebook", "Baidu", "Apple"];
const CATEGORIES: u64 = 5;
const FIRST_ID: u64 = 1000;
const BASE_TIME_MS: i64 = 1_000_000; pub const SENTINEL_ID: u64 = 1;
const _: () = assert!(SENTINEL_ID < FIRST_ID);
pub const SENTINEL_AHEAD_MS: i64 = 10_000_000;
fn event(n: u64, seed: u64, step_ms: i64, persons: u64, auctions: u64) -> (String, J, char) {
let mut r = Lcg(seed ^ (n.wrapping_mul(0x9E3779B97F4A7C15)));
r.next();
let date_time = BASE_TIME_MS + (n as i64) * step_ms;
match n % 50 {
0 => {
let id = FIRST_ID + persons;
let key = format!("p{id}");
let payload = json!({
"type": "person",
"id": id,
"name": format!("person-{id}"),
"emailAddress": format!("person-{id}@example.com"),
"creditCard": format!("{:04} {:04} {:04} {:04}", r.below(10000), r.below(10000), r.below(10000), r.below(10000)),
"city": CITIES[r.below(CITIES.len() as u64) as usize],
"state": US_STATES[r.below(US_STATES.len() as u64) as usize],
"dateTime": date_time,
});
(key, payload, 'p')
}
1..=3 => {
let id = FIRST_ID + auctions;
let key = format!("a{id}");
let initial = 1 + r.below(1000);
let seller = FIRST_ID + r.below(persons.max(1));
let payload = json!({
"type": "auction",
"id": id,
"itemName": format!("item-{id}"),
"initialBid": initial,
"reserve": initial + r.below(1000),
"dateTime": date_time,
"expires": date_time + 10_000 + r.below(100_000) as i64,
"seller": seller,
"category": r.below(CATEGORIES),
});
(key, payload, 'a')
}
_ => {
let auction = if r.below(2) == 0 {
FIRST_ID + auctions.saturating_sub(1)
} else {
FIRST_ID + r.below(auctions.max(1))
};
let bidder = FIRST_ID + r.below(persons.max(1));
let key = format!("a{auction}");
let payload = json!({
"type": "bid",
"auction": auction,
"bidder": bidder,
"price": 1 + r.below(10_000),
"channel": CHANNELS[r.below(CHANNELS.len() as u64) as usize],
"dateTime": date_time,
});
(key, payload, 'b')
}
}
}
pub fn spawn(cfg: NexmarkCfg) -> std::thread::JoinHandle<()> {
std::thread::spawn(move || {
if cfg.delay_ms > 0 {
std::thread::sleep(Duration::from_millis(cfg.delay_ms));
}
let producer: BaseProducer = match ClientConfig::new()
.set("bootstrap.servers", &cfg.brokers)
.set("enable.idempotence", "true")
.set("queue.buffering.max.messages", "1000000")
.create()
{
Ok(p) => p,
Err(e) => {
eprintln!("nexmark[{}]: producer create failed: {e}", cfg.dataset);
return;
}
};
let started = std::time::Instant::now();
let (mut persons, mut auctions) = (0u64, 0u64);
for n in 0..cfg.events {
let (key, payload, kind) = event(n, cfg.seed, cfg.step_ms, persons, auctions);
match kind {
'p' => persons += 1,
'a' => auctions += 1,
_ => {}
}
let bytes = serde_json::to_vec(&payload).expect("nexmark event serializes");
let mut rec = BaseRecord::to(&cfg.topic).key(&key).payload(&bytes);
loop {
match producer.send(rec) {
Ok(()) => break,
Err((_, r)) => {
producer.poll(Duration::from_millis(2));
rec = r;
}
}
}
producer.poll(Duration::ZERO);
if cfg.rate > 0 && n % 500 == 0 {
let due = started.elapsed().as_secs_f64() * cfg.rate as f64;
if (n as f64) > due {
std::thread::sleep(Duration::from_millis(
(((n as f64 - due) / cfg.rate as f64) * 1000.0) as u64 + 1,
));
}
}
}
let _ = producer.flush(Duration::from_secs(60));
println!(
"nexmark[{}]: produced {} real events in {:.1}s",
cfg.dataset,
cfg.events,
started.elapsed().as_secs_f64()
);
if cfg.tail {
let _ = producer.flush(Duration::from_secs(60));
std::thread::sleep(Duration::from_millis(cfg.tail_delay_ms));
let t = BASE_TIME_MS + (cfg.events as i64) * cfg.step_ms + SENTINEL_AHEAD_MS;
let sentinels = [
json!({"type": "person", "id": SENTINEL_ID, "name": "sentinel", "emailAddress": "s@s", "creditCard": "0", "city": "Nowhere", "state": "XX", "dateTime": t}),
json!({"type": "auction", "id": SENTINEL_ID, "itemName": "sentinel", "initialBid": 0, "reserve": 0, "dateTime": t, "expires": t, "seller": SENTINEL_ID, "category": 0}),
json!({"type": "bid", "auction": SENTINEL_ID, "bidder": SENTINEL_ID, "price": 0, "channel": "sentinel", "dateTime": t}),
];
for p in 0..cfg.partitions {
for payload in &sentinels {
let bytes = serde_json::to_vec(payload).expect("sentinel serializes");
let mut rec = BaseRecord::to(&cfg.topic).key("sentinel").payload(&bytes).partition(p);
loop {
match producer.send(rec) {
Ok(()) => break,
Err((_, r)) => {
producer.poll(Duration::from_millis(2));
rec = r;
}
}
}
}
}
}
let _ = producer.flush(Duration::from_secs(60));
println!(
"nexmark[{}]: generated {} events → {} ({} persons, {} auctions, {} bids) in {:.1}s",
cfg.dataset,
cfg.events,
cfg.topic,
persons,
auctions,
cfg.events - persons - auctions,
started.elapsed().as_secs_f64()
);
})
}
#[cfg(test)]
mod tests {
use super::*;
fn run(n_events: u64, seed: u64) -> Vec<(String, J, char)> {
let (mut persons, mut auctions) = (0u64, 0u64);
(0..n_events)
.map(|n| {
let e = event(n, seed, 10, persons, auctions);
match e.2 {
'p' => persons += 1,
'a' => auctions += 1,
_ => {}
}
e
})
.collect()
}
#[test]
fn deterministic_same_seed_same_stream() {
let a = run(500, 42);
let b = run(500, 42);
assert_eq!(format!("{a:?}"), format!("{b:?}"));
let c = run(500, 43);
assert_ne!(format!("{a:?}"), format!("{c:?}"), "different seed must differ");
}
#[test]
fn proportions_and_referential_integrity() {
let evs = run(5000, 42);
let (mut p, mut a, mut b) = (0u64, 0u64, 0u64);
let mut person_ids = std::collections::HashSet::new();
let mut auction_ids = std::collections::HashSet::new();
for (key, ev, kind) in &evs {
match kind {
'p' => {
p += 1;
person_ids.insert(ev["id"].as_u64().unwrap());
assert!(key.starts_with('p'));
}
'a' => {
a += 1;
auction_ids.insert(ev["id"].as_u64().unwrap());
assert!(
person_ids.contains(&ev["seller"].as_u64().unwrap()),
"auction seller must exist"
);
assert!(key.starts_with('a'));
}
_ => {
b += 1;
assert!(
auction_ids.contains(&ev["auction"].as_u64().unwrap()),
"bid auction must exist"
);
assert!(
person_ids.contains(&ev["bidder"].as_u64().unwrap()),
"bidder must exist"
);
assert_eq!(
*key,
format!("a{}", ev["auction"].as_u64().unwrap()),
"bids keyed by auction"
);
}
}
}
assert_eq!((p, a, b), (5000 / 50, 3 * 5000 / 50, 46 * 5000 / 50));
}
#[test]
fn event_time_is_monotonic_with_step() {
let evs = run(200, 7);
for (i, (_, ev, _)) in evs.iter().enumerate() {
assert_eq!(ev["dateTime"].as_i64().unwrap(), BASE_TIME_MS + (i as i64) * 10);
}
}
#[test]
fn golden_values_pin_the_deterministic_stream() {
let evs = run(52, 42);
let pin = |i: usize, key: &str, kind: char, json: &str| {
assert_eq!(evs[i].0, key, "event {i} key");
assert_eq!(evs[i].2, kind, "event {i} kind");
assert_eq!(serde_json::to_string(&evs[i].1).unwrap(), json, "event {i} payload");
};
pin(
0,
"p1000",
'p',
r#"{"city":"Boise","creditCard":"2172 7803 7043 3278","dateTime":1000000,"emailAddress":"person-1000@example.com","id":1000,"name":"person-1000","state":"AZ","type":"person"}"#,
);
pin(
1,
"a1000",
'a',
r#"{"category":3,"dateTime":1000010,"expires":1064796,"id":1000,"initialBid":166,"itemName":"item-1000","reserve":533,"seller":1000,"type":"auction"}"#,
);
pin(
4,
"a1002",
'b',
r#"{"auction":1002,"bidder":1000,"channel":"Facebook","dateTime":1000040,"price":9312,"type":"bid"}"#,
);
pin(
50,
"p1001",
'p',
r#"{"city":"Sacramento","creditCard":"0905 6287 0028 8090","dateTime":1000500,"emailAddress":"person-1001@example.com","id":1001,"name":"person-1001","state":"NV","type":"person"}"#,
);
}
#[test]
fn pacing_config_defaults_are_safe() {
let cfg = parse("d", &serde_json::json!({"connector": "nexmark", "topic": "t"}), "b")
.unwrap()
.unwrap();
assert_eq!(
(cfg.rate, cfg.delay_ms, cfg.events, cfg.seed, cfg.step_ms),
(0, 0, 100_000, 42, 10)
);
}
#[test]
fn parse_recognizes_only_nexmark_datasets() {
assert!(parse("d", &json!({"topic": "t"}), "b:9092").is_none());
let cfg = parse(
"d",
&json!({"connector": "nexmark", "topic": "t", "events": 5, "seed": 9, "rate": 100}),
"b:9092",
)
.unwrap()
.unwrap();
assert_eq!(
(cfg.events, cfg.seed, cfg.rate, cfg.partitions, cfg.delay_ms),
(5, 9, 100, 4, 0)
);
assert!(cfg.tail, "tail sentinels default ON");
assert_eq!(cfg.tail_delay_ms, 8_000, "quiesce default");
assert!(
parse("d", &json!({"connector": "nexmark"}), "b").unwrap().is_err(),
"topic required"
);
}
}