magnetar/partitioned_producer.rs
1// SPDX-License-Identifier: Apache-2.0
2
3//! Partition-aware producer.
4//!
5//! Mirrors Java's `PartitionedProducerImpl`. On `create()` the builder queries the broker for
6//! the topic's partition count via `CommandPartitionedTopicMetadata`. If the count is `> 1`
7//! it opens one child [`magnetar_runtime_tokio::Producer`] per partition (`<topic>-partition-N`)
8//! and routes user sends to the appropriate child via a configurable routing strategy.
9//! Otherwise it falls back to a single producer on the original topic.
10
11use std::sync::Arc;
12use std::sync::atomic::{AtomicU64, Ordering};
13use std::time::Duration;
14
15use bytes::Bytes;
16use magnetar_proto::types::CompressionKind;
17use magnetar_proto::{CreateProducerRequest, MessageId, pb};
18use magnetar_runtime_tokio::Producer;
19use tokio::sync::Notify;
20
21use crate::auto_update_task::{AutoUpdateTask, spawn_auto_update_task};
22use crate::client::{OutgoingMessage, PulsarError};
23use crate::{Engine, PulsarClient, TokioEngine};
24
25/// How a [`PartitionedProducer`] picks the partition for an outgoing message.
26#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
27pub enum MessageRoutingMode {
28 /// Hash the message's partition key via Java's `String.hashCode()` (the
29 /// `HashingScheme.JavaStringHash` default), then `% partitions`. Falls back
30 /// to round-robin when no key is set or the key is empty. Wire-compatible
31 /// with Java's default routing: the same key on a Rust producer and a
32 /// Java producer lands on the same partition.
33 #[default]
34 KeyHashOrRoundRobin,
35 /// Always round-robin, ignoring any partition key.
36 RoundRobin,
37 /// Always route to a single partition (`single_partition_index`). Useful for ordered
38 /// streams that don't need parallelism.
39 SinglePartition(u32),
40}
41
42/// Partitioned-producer-bound counterpart to [`crate::MessageBuilder`]. Same chained
43/// setters; the terminal `.send().await` resolves the partition and dispatches.
44#[derive(Debug)]
45pub struct PartitionedMessageBuilder<'a, P: crate::ProducerApi = Producer> {
46 producer: &'a PartitionedProducer<P>,
47 msg: OutgoingMessage,
48}
49
50impl<P: crate::ProducerApi> PartitionedMessageBuilder<'_, P> {
51 /// See [`OutgoingMessage::key`].
52 #[must_use]
53 pub fn key(mut self, key: impl Into<String>) -> Self {
54 self.msg = self.msg.key(key);
55 self
56 }
57
58 /// See [`OutgoingMessage::ordering_key`].
59 #[must_use]
60 pub fn ordering_key(mut self, key: impl Into<Bytes>) -> Self {
61 self.msg = self.msg.ordering_key(key);
62 self
63 }
64
65 /// See [`OutgoingMessage::event_time_ms`].
66 #[must_use]
67 pub fn event_time_ms(mut self, ts: u64) -> Self {
68 self.msg = self.msg.event_time_ms(ts);
69 self
70 }
71
72 /// See [`OutgoingMessage::property`].
73 #[must_use]
74 pub fn property(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
75 self.msg = self.msg.property(key, value);
76 self
77 }
78
79 /// See [`OutgoingMessage::deliver_at_ms`].
80 #[must_use]
81 pub fn deliver_at_ms(mut self, ts_ms: i64) -> Self {
82 self.msg = self.msg.deliver_at_ms(ts_ms);
83 self
84 }
85
86 /// See [`OutgoingMessage::deliver_after_ms`]. The caller supplies
87 /// `now_ms` (sans-io, ADR-0011 invariant #3).
88 #[must_use]
89 pub fn deliver_after_ms(mut self, now_ms: i64, delay_ms: i64) -> Self {
90 self.msg = self.msg.deliver_after_ms(now_ms, delay_ms);
91 self
92 }
93
94 /// See [`OutgoingMessage::replication_clusters`].
95 #[must_use]
96 pub fn replication_clusters(mut self, clusters: Vec<String>) -> Self {
97 self.msg = self.msg.replication_clusters(clusters);
98 self
99 }
100
101 /// See [`OutgoingMessage::disable_replication`].
102 #[must_use]
103 pub fn disable_replication(mut self) -> Self {
104 self.msg = self.msg.disable_replication();
105 self
106 }
107
108 /// See [`OutgoingMessage::txn`].
109 #[must_use]
110 pub fn txn(mut self, txn_id: magnetar_proto::TxnId) -> Self {
111 self.msg = self.msg.txn(txn_id);
112 self
113 }
114
115 /// Set the payload bytes. See [`OutgoingMessage::value`].
116 #[must_use]
117 pub fn value(mut self, payload: impl Into<Bytes>) -> Self {
118 self.msg = self.msg.value(payload);
119 self
120 }
121
122 /// Resolve the partition and dispatch. Returns the broker-assigned [`MessageId`].
123 pub async fn send(self) -> Result<MessageId, PulsarError> {
124 self.producer.send(self.msg).await
125 }
126}
127
128/// Plug a user-provided routing function in front of [`MessageRoutingMode`]. Mirrors
129/// Java's `MessageRouter` SPI — when set on the builder, the function decides the
130/// partition for every outgoing message; the configured [`MessageRoutingMode`] is
131/// ignored. Use this for affinity routing rules (geo, tenant, schema-keyed) that don't
132/// fit the partition-key-hash mould.
133///
134/// The callback runs on the send path — keep it fast and non-blocking. The framework
135/// clamps the returned index into `[0, partitions)` so out-of-range values can't crash
136/// the producer.
137pub trait MessageRouter: Send + Sync + std::fmt::Debug {
138 /// Pick a partition index in `[0, partitions)` for `msg`.
139 fn route(&self, msg: &crate::OutgoingMessage, partitions: usize) -> usize;
140}
141
142/// Bit-for-bit port of Apache Pulsar's `Murmur3_32Hash.makeHash(byte[])`
143/// ([`Murmur3_32Hash.java`]). Used by [`Murmur3HashHasher`] so cross-language consumers
144/// (Java, C++, Go) see identical routing for the same key.
145///
146/// Returns a non-negative 31-bit value — the Java implementation masks with
147/// `Integer.MAX_VALUE` before returning.
148///
149/// [`Murmur3_32Hash.java`]: https://github.com/apache/pulsar/blob/master/pulsar-common/src/main/java/org/apache/pulsar/common/util/Murmur3_32Hash.java
150#[must_use]
151pub fn murmur3_32_hash(bytes: &[u8]) -> u32 {
152 const C1: u32 = 0xcc9e_2d51;
153 const C2: u32 = 0x1b87_3593;
154 const SEED: u32 = 0;
155
156 let len = bytes.len();
157 let mut h1: u32 = SEED;
158
159 let mix_k1 = |mut k1: u32| -> u32 {
160 k1 = k1.wrapping_mul(C1);
161 k1 = k1.rotate_left(15);
162 k1 = k1.wrapping_mul(C2);
163 k1
164 };
165
166 let (chunks, remainder) = bytes.as_chunks::<4>();
167 for chunk in chunks {
168 // Java's `ByteBuffer.LITTLE_ENDIAN.getInt()` reads four bytes little-endian.
169 let k1 = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
170 let k1 = mix_k1(k1);
171 h1 ^= k1;
172 h1 = h1.rotate_left(13);
173 h1 = h1.wrapping_mul(5).wrapping_add(0xe654_6b64);
174 }
175
176 // Tail.
177 let mut k1: u32 = 0;
178 for (i, byte) in remainder.iter().enumerate() {
179 k1 ^= u32::from(*byte) << (i * 8);
180 }
181 h1 ^= mix_k1(k1);
182
183 // Finalisation: XOR length, then `fmix`.
184 h1 ^= len as u32;
185 h1 ^= h1 >> 16;
186 h1 = h1.wrapping_mul(0x85eb_ca6b);
187 h1 ^= h1 >> 13;
188 h1 = h1.wrapping_mul(0xc2b2_ae35);
189 h1 ^= h1 >> 16;
190
191 // Mirror Java's `& Integer.MAX_VALUE` mask so the value fits into a non-negative
192 // signed int32 — matches `Murmur3Hash32.makeHash` and `Murmur3_32Hash.makeHash`.
193 h1 & 0x7FFF_FFFF
194}
195
196/// Bit-for-bit port of `String.hashCode() & Integer.MAX_VALUE`. Iterates over UTF-16
197/// code units (matching Java's `char`) so non-BMP code points hash identically to the
198/// JDK. ASCII strings short-circuit through the byte path.
199///
200/// Used by [`JavaStringHashHasher`].
201#[must_use]
202pub fn java_string_hash(key: &str) -> u32 {
203 let mut h: u32 = 0;
204 if key.is_ascii() {
205 for byte in key.bytes() {
206 h = h.wrapping_mul(31).wrapping_add(u32::from(byte));
207 }
208 } else {
209 for code_unit in key.encode_utf16() {
210 h = h.wrapping_mul(31).wrapping_add(u32::from(code_unit));
211 }
212 }
213 h & 0x7FFF_FFFF
214}
215
216/// Pick the partition by hashing the message's UTF-8-encoded partition key with
217/// [`murmur3_32_hash`] (Apache Pulsar `Murmur3_32Hash`, seed `0`), then `hash %
218/// partitions`. Falls back to round-robin via [`OutgoingMessage::key`] being `None` or
219/// empty.
220///
221/// Wire-compatible with Java's `HashingScheme.Murmur3_32Hash` so Java, C++, Go, and
222/// magnetar producers route the same key to the same partition.
223#[derive(Debug, Default, Clone, Copy)]
224pub struct Murmur3HashHasher;
225
226impl MessageRouter for Murmur3HashHasher {
227 fn route(&self, msg: &crate::OutgoingMessage, partitions: usize) -> usize {
228 partition_for_key(msg.key.as_deref(), partitions, |k| {
229 murmur3_32_hash(k.as_bytes())
230 })
231 }
232}
233
234/// Pick the partition with [`java_string_hash`] (Java `String.hashCode()` semantics),
235/// then `hash % partitions`. Falls back to round-robin when no key is set.
236///
237/// Wire-compatible with Java's default `HashingScheme.JavaStringHash`.
238#[derive(Debug, Default, Clone, Copy)]
239pub struct JavaStringHashHasher;
240
241impl MessageRouter for JavaStringHashHasher {
242 fn route(&self, msg: &crate::OutgoingMessage, partitions: usize) -> usize {
243 partition_for_key(msg.key.as_deref(), partitions, java_string_hash)
244 }
245}
246
247/// Shared "keyed hash, fall back to a sticky default" routing helper. The fallback
248/// returns partition `0`; the surrounding [`PartitionedProducer`] is responsible for
249/// running round-robin when no router is installed. When a router *is* installed it
250/// overrides the configured [`MessageRoutingMode`] entirely (mirrors Java
251/// `ProducerBuilder#messageRouter`), so we cannot rotate through the cursor here —
252/// instead we sticky-route to partition `0`, matching Java's `RoundRobinPartitionMessageRouter`
253/// behaviour when the key is null and batching keeps a key-affine sticky partition.
254fn partition_for_key<F>(key: Option<&str>, partitions: usize, hash: F) -> usize
255where
256 F: FnOnce(&str) -> u32,
257{
258 if partitions == 0 {
259 return 0;
260 }
261 match key {
262 Some(k) if !k.is_empty() => (hash(k) as usize) % partitions,
263 _ => 0,
264 }
265}
266
267/// Partition-aware producer.
268///
269/// Generic over `P: ProducerApi` per ADR-0026 §D1 (default
270/// `magnetar_runtime_tokio::Producer`). The general inherent impl
271/// dispatches `send` / `flush` / `close` / `stats` through `ProducerApi`;
272/// tokio-only specialised methods (`refresh_partitions`,
273/// `last_sequence_id_published`, batch counters) live in the
274/// `impl PartitionedProducer<Producer>` specialised block below.
275#[derive(Debug)]
276pub struct PartitionedProducer<P: crate::ProducerApi = Producer> {
277 partitions: Vec<P>,
278 base_topic: String,
279 routing: MessageRoutingMode,
280 /// Optional custom router. When set, takes precedence over [`Self::routing`] for
281 /// every send.
282 router: Option<std::sync::Arc<dyn MessageRouter>>,
283 cursor: AtomicU64,
284 /// Optional background partition-watcher task. `Some` when the builder configured
285 /// [`PartitionedProducerBuilder::auto_update_partitions_interval`], `None`
286 /// otherwise (default). The task is a pure timer that signals
287 /// [`Self::partitions_changed_notify`] every interval; the actual
288 /// `partitions_for_topic` call is driven by user code via
289 /// [`Self::refresh_partitions`]. Dropping the [`PartitionedProducer`] aborts the
290 /// task.
291 auto_update: Option<Arc<AutoUpdateTask>>,
292}
293
294impl<P: crate::ProducerApi> PartitionedProducer<P> {
295 /// Base topic name (without the `-partition-N` suffix).
296 #[must_use]
297 pub fn topic(&self) -> &str {
298 &self.base_topic
299 }
300
301 /// Number of child producers (1 for non-partitioned topics).
302 #[must_use]
303 pub fn partitions(&self) -> usize {
304 self.partitions.len()
305 }
306
307 /// Borrow the underlying per-partition producers. Useful for advanced operations
308 /// like per-partition flush.
309 #[must_use]
310 pub fn child_producers(&self) -> &[P] {
311 &self.partitions
312 }
313
314 /// Publish a message, routing it to one of the underlying producers per the configured
315 /// [`MessageRoutingMode`] (or the custom `MessageRouter` when one was installed on the
316 /// builder). Returns the broker-assigned message id (the routing layer is transparent
317 /// — the id has a `partition` filled in by the broker).
318 ///
319 /// # Errors
320 /// - [`PulsarError::Other`] (stringified from the runtime's `ProducerApi::Error`) on wire
321 /// failure.
322 pub async fn send(&self, msg: OutgoingMessage) -> Result<MessageId, PulsarError> {
323 let idx = self.pick_partition(&msg);
324 let producer = &self.partitions[idx];
325 crate::ProducerApi::send(producer, msg)
326 .await
327 .map_err(|err| PulsarError::Other(format!("send: {err}")))
328 }
329
330 /// Start a Java-symmetric `MessageBuilder` chain that ends with `.send().await`. The
331 /// routing decision happens on `send` based on the constructed `OutgoingMessage`, so
332 /// `.key(..)` participates in `MessageRoutingMode::KeyHashOrRoundRobin` and any
333 /// installed `MessageRouter` sees the full message.
334 #[must_use]
335 pub fn new_message(&self) -> PartitionedMessageBuilder<'_, P> {
336 PartitionedMessageBuilder {
337 producer: self,
338 msg: OutgoingMessage::default(),
339 }
340 }
341
342 fn pick_partition(&self, msg: &OutgoingMessage) -> usize {
343 let n = self.partitions.len();
344 if n == 0 {
345 return 0;
346 }
347 if let Some(router) = &self.router {
348 // Clamp into range so an out-of-range router can't crash the producer.
349 return router.route(msg, n).min(n - 1);
350 }
351 let key = msg.key.as_deref();
352 match self.routing {
353 MessageRoutingMode::SinglePartition(p) => (p as usize).min(n - 1),
354 MessageRoutingMode::RoundRobin => {
355 let prev = self.cursor.fetch_add(1, Ordering::Relaxed);
356 (prev as usize) % n
357 }
358 MessageRoutingMode::KeyHashOrRoundRobin => match key {
359 // Java parity: `HashingScheme.JavaStringHash` —
360 // `String.hashCode() & Integer.MAX_VALUE`, masked into a
361 // non-negative i32. The earlier implementation used Rust's
362 // `DefaultHasher` (SipHash with a process-randomised seed),
363 // which broke cross-language key affinity: the same key
364 // routed to a different partition from a Java producer,
365 // defeating the whole point of keyed partitioning. See R2/F3
366 // for the regression. The `java_string_hash` helper is the
367 // same one wired into [`JavaStringHashHasher`].
368 Some(k) if !k.is_empty() => (java_string_hash(k) as usize) % n,
369 _ => {
370 let prev = self.cursor.fetch_add(1, Ordering::Relaxed);
371 (prev as usize) % n
372 }
373 },
374 }
375 }
376
377 /// Aggregate cumulative stats across all child producers (issue #347).
378 /// Thin wrapper over [`magnetar_proto::ProducerStats::fold`] — collects
379 /// each child's `(stats(), send_latency_histogram())` snapshot and folds
380 /// them per that function's documented per-field rule: the four
381 /// cumulative totals + `pending_queue_size` sum; `msgs_per_sec` /
382 /// `bytes_per_sec` sum as f64 (fan-in throughput); `send_latency_max_ms`
383 /// is the exact max; `send_latency_p50_ms` / `send_latency_p99_ms` are
384 /// recomputed from a REAL merge of every child's send-latency histogram
385 /// — summing or maxing percentiles directly (the previous
386 /// implementation's bug: those five fields were silently left at their
387 /// `ProducerStats::default()` zero) is not statistically sound.
388 ///
389 /// The two rate fields are populated by the client-wide sweep armed with
390 /// [`crate::ClientBuilder::stats_interval`], which reaches every partition
391 /// because it ticks each slot on the connection rather than fanning out
392 /// from here (ADR-0089 — Java's `PartitionedProducerImpl.getStats()` has no
393 /// fan-out either, and one clock ticking every child is what makes the f64
394 /// sum well-defined). With that knob unset they stay caller-driven and
395 /// therefore `0.0`; see
396 /// [`magnetar_proto::producer::ProducerState::record_rate_window`].
397 ///
398 /// A partition added mid-window by partition growth is seeded at its own
399 /// creation, so it contributes a full snapshot of its counters immediately
400 /// but `0.0` to the rate fields for its first full interval — the window
401 /// needs a baseline first. Java's recorders behave identically.
402 #[must_use]
403 pub fn aggregate_stats(&self) -> magnetar_proto::ProducerStats {
404 let children: Vec<_> = self
405 .partitions
406 .iter()
407 .map(|p| {
408 (
409 crate::ProducerApi::stats(p),
410 crate::ProducerApi::send_latency_histogram(p),
411 )
412 })
413 .collect();
414 magnetar_proto::ProducerStats::fold(children)
415 }
416
417 /// Close every child producer. Returns the first error encountered.
418 ///
419 /// # Errors
420 /// - [`PulsarError::Other`] (stringified) on the first child failure.
421 pub async fn close(self) -> Result<(), PulsarError> {
422 let mut first_err: Result<(), PulsarError> = Ok(());
423 for p in self.partitions {
424 if let Err(e) = crate::ProducerApi::close_owned(p).await
425 && first_err.is_ok()
426 {
427 first_err = Err(PulsarError::Other(format!("close: {e}")));
428 }
429 }
430 first_err
431 }
432
433 /// Flush every child producer in parallel. Mirrors Java
434 /// `Producer#flushAsync` semantics — resolves once each per-partition pending queue
435 /// drains. Returns the first error encountered.
436 ///
437 /// # Errors
438 /// - [`PulsarError::Other`] (stringified) on the first child failure.
439 pub async fn flush(&self) -> Result<(), PulsarError> {
440 let mut first_err: Result<(), PulsarError> = Ok(());
441 for p in &self.partitions {
442 if let Err(e) = crate::ProducerApi::flush(p).await
443 && first_err.is_ok()
444 {
445 first_err = Err(PulsarError::Other(format!("flush: {e}")));
446 }
447 }
448 first_err
449 }
450
451 /// `true` while every child producer reports the underlying connection is up. Mirrors
452 /// Java `Producer#isConnected` at the partitioned scope — Java returns true iff every
453 /// partition's underlying producer is connected.
454 #[must_use]
455 pub fn is_connected(&self) -> bool {
456 self.partitions.iter().all(crate::ProducerApi::is_connected)
457 }
458
459 /// Earliest wall-clock disconnect timestamp across all child producers, or `None` if
460 /// no child has ever disconnected. Useful for "when did we last see a connection
461 /// drop?" health probes.
462 #[must_use]
463 pub fn last_disconnected_timestamp(&self) -> Option<std::time::SystemTime> {
464 self.partitions
465 .iter()
466 .filter_map(crate::ProducerApi::last_disconnected_timestamp)
467 .min()
468 }
469
470 /// `true` once every child producer is closed. Mirrors Java `Producer#isClosed` at the
471 /// partitioned scope. Pair with [`Self::is_connected`] for the live test — `is_closed`
472 /// only flips after a terminal close, not on transient disconnects.
473 #[must_use]
474 pub fn is_closed(&self) -> bool {
475 self.partitions.iter().all(crate::ProducerApi::is_closed)
476 }
477
478 /// Max `last_sequence_id` across every child producer (i.e. the largest sequence id
479 /// this client has pushed onto the wire on any partition). Returns `-1` when no
480 /// partition has sent yet. Useful for at-least-once resume-on-restart at the
481 /// partitioned scope. Mirrors Java `Producer#getLastSequenceId` aggregated.
482 #[must_use]
483 pub fn last_sequence_id(&self) -> i64 {
484 self.partitions
485 .iter()
486 .map(crate::ProducerApi::last_sequence_id)
487 .max()
488 .unwrap_or(-1)
489 }
490
491 /// Returns `true` if a background partition-watcher was spawned for this
492 /// producer (i.e.
493 /// [`PartitionedProducerBuilder::auto_update_partitions_interval`] was set on
494 /// the builder). Defaults to `false` — current Java-parity behaviour when the
495 /// user did not opt in.
496 #[must_use]
497 pub fn has_auto_update_partitions(&self) -> bool {
498 self.auto_update.is_some()
499 }
500
501 /// Most recent partition count observed by the background partition watcher.
502 /// `None` when
503 /// [`PartitionedProducerBuilder::auto_update_partitions_interval`] was not set
504 /// (no watcher spawned). Mirrors the read side of Java's
505 /// `ProducerBuilder#autoUpdatePartitionsInterval` behaviour — Java rebuilds
506 /// internally; we expose the observation so callers can react.
507 #[must_use]
508 pub fn observed_partitions(&self) -> Option<u32> {
509 self.auto_update
510 .as_ref()
511 .map(|t| t.observed_partitions.load(Ordering::Relaxed))
512 }
513
514 /// Monotonic count of partition-change events observed by the background
515 /// watcher. Returns `None` when no watcher was configured. The counter starts
516 /// at `0` and is bumped every time [`Self::refresh_partitions`] detects a
517 /// different partition count than the previous observation.
518 #[must_use]
519 pub fn partition_change_count(&self) -> Option<u64> {
520 self.auto_update
521 .as_ref()
522 .map(|t| t.change_count.load(Ordering::Relaxed))
523 }
524
525 /// `Arc<Notify>` signalled by the background partition-watcher on every timer
526 /// tick (i.e. every `auto_update_partitions_interval`) and on every observed
527 /// partition-count change driven by [`Self::refresh_partitions`]. Returns
528 /// `None` when no watcher was configured. Callers may `await` `notified()` on
529 /// the returned handle to react to ticks without polling
530 /// [`Self::partition_change_count`].
531 #[must_use]
532 pub fn partitions_changed_notify(&self) -> Option<Arc<Notify>> {
533 self.auto_update.as_ref().map(|t| t.changed.clone())
534 }
535}
536
537/// Tokio-engine-specific `PartitionedProducer` methods that depend on
538/// either (a) `PulsarClient<TokioEngine>` (e.g. `refresh_partitions`
539/// which calls `client.partitions_for_topic`) or (b) Producer helpers
540/// not yet on `ProducerApi` (`last_sequence_id_published`,
541/// `batch_len`, `batch_bytes`, `pending_count`). Each of these
542/// methods can be lifted once the matching helper lands on
543/// `ProducerApi` / a future `EngineClient` trait.
544impl PartitionedProducer<Producer> {
545 /// Max `last_sequence_id_published` across every child producer. Returns `-1` when no
546 /// partition has been acknowledged yet. Mirrors Java
547 /// `Producer#getLastSequenceIdPublished` aggregated.
548 #[must_use]
549 pub fn last_sequence_id_published(&self) -> i64 {
550 self.partitions
551 .iter()
552 .map(magnetar_runtime_tokio::Producer::last_sequence_id_published)
553 .max()
554 .unwrap_or(-1)
555 }
556
557 /// Sum of in-flight sends across every child producer.
558 #[must_use]
559 pub fn pending_count(&self) -> usize {
560 self.partitions
561 .iter()
562 .map(magnetar_runtime_tokio::Producer::pending_count)
563 .sum()
564 }
565
566 /// Sum of batch-buffered messages across every child producer.
567 #[must_use]
568 pub fn batch_len(&self) -> usize {
569 self.partitions
570 .iter()
571 .map(magnetar_runtime_tokio::Producer::batch_len)
572 .sum()
573 }
574
575 /// Sum of batch-buffered payload bytes across every child producer.
576 #[must_use]
577 pub fn batch_bytes(&self) -> usize {
578 self.partitions
579 .iter()
580 .map(magnetar_runtime_tokio::Producer::batch_bytes)
581 .sum()
582 }
583
584 /// Query the broker for the current partition count of the topic this producer
585 /// was opened against, and update [`Self::observed_partitions`] /
586 /// [`Self::partition_change_count`] in place if the count differs from the
587 /// last observation.
588 ///
589 /// This is the user-driven half of the
590 /// [`PartitionedProducerBuilder::auto_update_partitions_interval`] machinery:
591 /// the timer task signals [`Self::partitions_changed_notify`]; the user calls
592 /// this method in response (or independently) to actually refresh the count.
593 /// Returns the freshly-observed count on success, or `Ok(None)` if no watcher
594 /// was configured (no topic recorded). Errors are surfaced via [`PulsarError`].
595 ///
596 /// **Note**: this method only updates the observed count. It does *not* itself
597 /// add new child producers to match a grown partition count — that is a
598 /// follow-up. Callers that need to expand the producer set can detect the
599 /// change via [`Self::observed_partitions`] / [`Self::partitions`] divergence
600 /// and rebuild the producer.
601 ///
602 /// # Errors
603 ///
604 /// Surfaces [`PulsarError::Client`] when the broker metadata lookup fails.
605 pub async fn refresh_partitions(
606 &self,
607 client: &PulsarClient,
608 ) -> Result<Option<u32>, PulsarError> {
609 let Some(task) = self.auto_update.as_ref() else {
610 return Ok(None);
611 };
612 let count = client.partitions_for_topic(&task.topic).await?;
613 // Atomic swap-then-compare. See multi_topics.rs for the rationale.
614 let prev = task.observed_partitions.swap(count, Ordering::Relaxed);
615 if prev != count {
616 task.change_count.fetch_add(1, Ordering::Relaxed);
617 task.changed.notify_waiters();
618 }
619 Ok(Some(count))
620 }
621}
622
623/// Builder for [`PartitionedProducer`]. Mirrors Java's `ProducerBuilder` at the partitioned
624/// layer.
625///
626/// Engine-generic: the type parameter `E: Engine` (defaults to
627/// [`crate::TokioEngine`]) selects the per-partition child producer type
628/// via the engine-side [`crate::CreateProducerApi`] +
629/// [`crate::BrokerMetadataApi`] extension traits. The encryptor slot is
630/// engine-typed via
631/// [`crate::MessageEncryptorApi`] (tokio plugs in
632/// `Arc<dyn magnetar_runtime_tokio::MessageEncryptor>`; moonpool plugs in
633/// `Arc<dyn magnetar_runtime_moonpool::MessageEncryptor>` now that the
634/// moonpool engine ships the PIP-4 bridge).
635pub struct PartitionedProducerBuilder<'a, E: Engine = TokioEngine> {
636 client: &'a PulsarClient<E>,
637 topic: String,
638 name: Option<String>,
639 compression: CompressionKind,
640 enable_batching: bool,
641 enable_chunking: bool,
642 max_batch_size_bytes: usize,
643 max_messages_in_batch: usize,
644 routing: MessageRoutingMode,
645 initial_sequence_id: Option<u64>,
646 access_mode: pb::ProducerAccessMode,
647 producer_metadata: Vec<(String, String)>,
648 send_timeout: Option<std::time::Duration>,
649 batching_max_publish_delay: Option<std::time::Duration>,
650 schema: Option<pb::Schema>,
651 encryptor: Option<<E as crate::MessageEncryptorApi>::Encryptor>,
652 router: Option<std::sync::Arc<dyn MessageRouter>>,
653 auto_update_partitions_interval: Option<Duration>,
654}
655
656impl<E: Engine> std::fmt::Debug for PartitionedProducerBuilder<'_, E> {
657 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
658 f.debug_struct("PartitionedProducerBuilder")
659 .field("topic", &self.topic)
660 .field("name", &self.name)
661 .field("routing", &self.routing)
662 .finish()
663 }
664}
665
666impl<'a, E: Engine> PartitionedProducerBuilder<'a, E> {
667 pub(crate) fn new(client: &'a PulsarClient<E>, topic: String) -> Self {
668 Self {
669 client,
670 topic,
671 name: None,
672 compression: CompressionKind::None,
673 enable_batching: false,
674 enable_chunking: false,
675 max_batch_size_bytes: 128 * 1024,
676 max_messages_in_batch: 1000,
677 routing: MessageRoutingMode::default(),
678 initial_sequence_id: None,
679 access_mode: pb::ProducerAccessMode::Shared,
680 producer_metadata: Vec::new(),
681 // Inherit the canonical Java-parity default (Some(30s), ADR-0072) from
682 // CreateProducerRequest so the partitioned builder never silently pins
683 // the old never-times-out semantics. `send_timeout(d)` overrides it.
684 send_timeout: CreateProducerRequest::default().send_timeout,
685 batching_max_publish_delay: None,
686 schema: None,
687 encryptor: None,
688 router: None,
689 auto_update_partitions_interval: None,
690 }
691 }
692
693 /// Install a custom [`MessageRouter`]. When set, the router overrides
694 /// [`Self::routing`] for every send. Mirrors Java
695 /// `ProducerBuilder#messageRouter(MessageRouter)`.
696 #[must_use]
697 pub fn message_router(mut self, router: std::sync::Arc<dyn MessageRouter>) -> Self {
698 self.router = Some(router);
699 self
700 }
701
702 /// Set the producer name advertised to the broker.
703 #[must_use]
704 pub fn name(mut self, name: impl Into<String>) -> Self {
705 self.name = Some(name.into());
706 self
707 }
708
709 /// Set the compression codec.
710 #[must_use]
711 pub fn compression(mut self, kind: CompressionKind) -> Self {
712 self.compression = kind;
713 self
714 }
715
716 /// Enable batching with the given limits.
717 #[must_use]
718 pub fn batching(mut self, max_messages: usize, max_bytes: usize) -> Self {
719 self.enable_batching = true;
720 self.max_messages_in_batch = max_messages;
721 self.max_batch_size_bytes = max_bytes;
722 self
723 }
724
725 /// Enable chunking for oversize messages.
726 #[must_use]
727 pub fn chunking(mut self, enable: bool) -> Self {
728 self.enable_chunking = enable;
729 self
730 }
731
732 /// Set the routing mode.
733 #[must_use]
734 pub fn routing(mut self, mode: MessageRoutingMode) -> Self {
735 self.routing = mode;
736 self
737 }
738
739 /// Set the initial sequence id (applied to every per-partition producer).
740 #[must_use]
741 pub fn initial_sequence_id(mut self, id: u64) -> Self {
742 self.initial_sequence_id = Some(id);
743 self
744 }
745
746 /// Producer access mode (`Shared` / `Exclusive` / `WaitForExclusive` /
747 /// `ExclusiveWithFencing`) — applied to every per-partition child producer.
748 #[must_use]
749 pub fn access_mode(mut self, mode: pb::ProducerAccessMode) -> Self {
750 self.access_mode = mode;
751 self
752 }
753
754 /// Appends a `(key, value)` entry to the broker-visible producer metadata, applied
755 /// to every per-partition child. Mirrors Java `ProducerBuilder#property` at the
756 /// partitioned scope.
757 #[must_use]
758 pub fn property(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
759 self.producer_metadata.push((key.into(), value.into()));
760 self
761 }
762
763 /// Mirrors Java `ProducerBuilder#sendTimeout` — applied to every per-partition child.
764 /// In-flight sends past their `enqueued_at + timeout` deadline resolve with a
765 /// synthetic `code=-1, message="send timeout"` `SendError`.
766 #[must_use]
767 pub fn send_timeout(mut self, timeout: std::time::Duration) -> Self {
768 self.send_timeout = Some(timeout);
769 self
770 }
771
772 /// Mirrors Java `ProducerBuilder#batchingMaxPublishDelay` — applied to every
773 /// per-partition child. With batching enabled, the state machine flushes any non-empty
774 /// batch whose oldest message has been waiting longer than `delay`.
775 #[must_use]
776 pub fn batching_max_publish_delay(mut self, delay: std::time::Duration) -> Self {
777 self.batching_max_publish_delay = Some(delay);
778 self
779 }
780
781 /// Advertise a schema on every per-partition `CommandProducer`.
782 #[must_use]
783 pub fn schema(mut self, schema: pb::Schema) -> Self {
784 self.schema = Some(schema);
785 self
786 }
787
788 /// Enable a background timer that signals every `interval`, intended to drive
789 /// re-checks of the topic's partition count. Mirrors Java
790 /// `ProducerBuilder#autoUpdatePartitionsInterval`.
791 ///
792 /// The internal timer task signals
793 /// [`PartitionedProducer::partitions_changed_notify`] on every tick. Callers
794 /// run [`PartitionedProducer::refresh_partitions`] in response to the signal
795 /// (or on their own cadence) to actually call
796 /// [`PulsarClient::partitions_for_topic`] — the timer itself is decoupled from
797 /// the client so the watcher stays compatible with the crate-wide
798 /// `#![forbid(unsafe_code)]` invariant.
799 ///
800 /// Default `None` — no timer is spawned and a [`PartitionedProducer`] over a
801 /// partitioned topic will not notice partitions added after construction. Pass
802 /// a non-zero `Duration` to opt in. The timer is aborted when the
803 /// [`PartitionedProducer`] is dropped.
804 ///
805 /// Setting a zero `interval` is treated as "disable" — same as the default.
806 #[must_use]
807 pub fn auto_update_partitions_interval(mut self, interval: Duration) -> Self {
808 self.auto_update_partitions_interval = if interval.is_zero() {
809 None
810 } else {
811 Some(interval)
812 };
813 self
814 }
815
816 /// Query partition count, then open one producer per partition. If the broker reports
817 /// `0` partitions, fall back to a single producer on the original topic.
818 ///
819 /// Dispatches through the engine-generic
820 /// [`crate::BrokerMetadataApi`] (partition count lookup) and
821 /// [`crate::CreateProducerApi`] (per-partition producer creation)
822 /// extension traits, so the same builder shape works for both the
823 /// tokio and moonpool engines.
824 ///
825 /// **PIP-4 encryption guardrail (BREAKING since the encryptor-storage lift).**
826 /// If [`Self::encryption`] was called on the per-engine specialisation,
827 /// `.create()` returns [`PulsarError::Other`] instead of silently opening
828 /// plaintext per-partition producers. The engine-generic dispatch does not
829 /// know how to thread an engine-typed encryptor through `open_producer`,
830 /// so the previous "silently drop the encryptor" behaviour was a footgun.
831 /// Use [`Self::create_with_encryption`] on the tokio specialisation
832 /// instead.
833 ///
834 /// # Errors
835 ///
836 /// - [`PulsarError::Other`] if an encryptor was configured via [`Self::encryption`] — call
837 /// `create_with_encryption()` instead.
838 /// - [`PulsarError::Other`] (stringified) on the broker metadata lookup or on a per-partition
839 /// producer open failure.
840 pub async fn create(
841 self,
842 ) -> Result<
843 PartitionedProducer<<E::ClientState as crate::CreateProducerApi>::Producer>,
844 PulsarError,
845 >
846 where
847 E::ClientState: crate::BrokerMetadataApi + crate::CreateProducerApi,
848 {
849 if self.encryptor.is_some() {
850 return Err(PulsarError::Other(
851 "PartitionedProducerBuilder::create() refuses a configured encryptor — \
852 use create_with_encryption() on the engine-specific builder \
853 (PIP-4 encryptors are engine-typed and cannot dispatch \
854 through the engine-generic CreateProducerApi)"
855 .to_owned(),
856 ));
857 }
858 let base_req = CreateProducerRequest {
859 topic: self.topic,
860 producer_name: self.name,
861 compression: self.compression,
862 enable_batching: self.enable_batching,
863 enable_chunking: self.enable_chunking,
864 max_batch_size_bytes: self.max_batch_size_bytes,
865 max_messages_in_batch: self.max_messages_in_batch,
866 schema: self.schema,
867 initial_sequence_id: self.initial_sequence_id,
868 access_mode: self.access_mode,
869 producer_metadata: self.producer_metadata,
870 send_timeout: self.send_timeout,
871 batching_max_publish_delay: self.batching_max_publish_delay,
872 };
873 let mut deadline =
874 crate::CreateProducerApi::new_producer_operation_deadline(&self.client.inner);
875 open_partitioned_with_metadata(
876 self.client,
877 base_req,
878 self.routing,
879 self.router,
880 self.auto_update_partitions_interval,
881 &mut deadline,
882 )
883 .await
884 }
885}
886
887/// Resolve partition metadata for `base_req.topic`, open one producer per
888/// resolved partition (or a single producer on the bare topic when `N == 0`),
889/// and wrap the result in a [`PartitionedProducer`].
890///
891/// Used only by [`PartitionedProducerBuilder::create`] — the explicit
892/// partitioned-producer entry point. A bare
893/// [`crate::builders::ProducerBuilder::create`] does NOT delegate here: per
894/// ADR-0051 it pre-checks the partition metadata and, on a partitioned topic,
895/// returns an actionable error pointing the caller at
896/// `client.partitioned_producer(...)` rather than silently fanning out (the
897/// rejected "auto-dispatch exactly like Java" option). Without that pre-check a
898/// bare `open_producer` on a partitioned topic surfaces as broker
899/// `NotAllowedError(22) "Found partitioned metadata for non-partitioned topic"`
900/// — the rough edge that drove ADR-0051.
901///
902/// On per-partition open failure every already-opened child is closed before
903/// the error propagates, so a partial fan-out never leaks producers on the
904/// broker side.
905pub(crate) async fn open_partitioned_with_metadata<E>(
906 client: &PulsarClient<E>,
907 base_req: CreateProducerRequest,
908 routing: MessageRoutingMode,
909 router: Option<Arc<dyn MessageRouter>>,
910 auto_update_partitions_interval: Option<Duration>,
911 deadline: &mut crate::OperationDeadline,
912) -> Result<PartitionedProducer<<E::ClientState as crate::CreateProducerApi>::Producer>, PulsarError>
913where
914 E: Engine,
915 E::ClientState: crate::BrokerMetadataApi + crate::CreateProducerApi,
916{
917 let base_topic = base_req.topic.clone();
918 let partitions_count = crate::BrokerMetadataApi::partitioned_topic_metadata_with_deadline(
919 &client.inner,
920 &base_topic,
921 deadline,
922 )
923 .await
924 .map_err(|err| PulsarError::Other(format!("partitioned_topic_metadata: {err}")))?;
925
926 let partition_topics: Vec<String> = if partitions_count == 0 {
927 vec![base_topic.clone()]
928 } else {
929 (0..partitions_count)
930 .map(|i| format!("{base_topic}-partition-{i}"))
931 .collect()
932 };
933
934 let mut child_producers: Vec<<E::ClientState as crate::CreateProducerApi>::Producer> =
935 Vec::with_capacity(partition_topics.len());
936 for child_topic in &partition_topics {
937 let mut req = base_req.clone();
938 req.topic = child_topic.clone();
939 let result =
940 crate::CreateProducerApi::open_producer_with_deadline(&client.inner, req, deadline)
941 .await;
942 match result {
943 Ok(p) => child_producers.push(p),
944 Err(e) => {
945 for p in child_producers {
946 let _ = crate::ProducerApi::close_owned(p).await;
947 }
948 return Err(PulsarError::Other(format!("open_producer: {e}")));
949 }
950 }
951 }
952
953 // Spawn the partition-watcher timer iff the builder configured a non-zero
954 // interval. The timer itself only emits ticks via `Notify`; callers drive
955 // the actual `partitions_for_topic` call via
956 // [`PartitionedProducer::refresh_partitions`] (the crate-wide
957 // `#![forbid(unsafe_code)]` rules out punning the `&PulsarClient` lifetime
958 // into a `'static` spawn).
959 let auto_update = auto_update_partitions_interval
960 .map(|interval| spawn_auto_update_task(base_topic.clone(), interval, partitions_count));
961
962 Ok(PartitionedProducer {
963 partitions: child_producers,
964 base_topic,
965 routing,
966 router,
967 cursor: AtomicU64::new(0),
968 auto_update,
969 })
970}
971
972/// Tokio-engine-specific `PartitionedProducerBuilder` methods that need
973/// the `open_producer_with(encryptor)` runtime carve-out (PIP-4 not yet
974/// wired on moonpool).
975impl PartitionedProducerBuilder<'_, TokioEngine> {
976 /// Configure PIP-4 end-to-end encryption (applied to every per-partition producer).
977 /// Tokio-engine-only — call [`Self::create_with_encryption`] to honor the
978 /// encryptor on the open path. The engine-generic [`Self::create`] ignores
979 /// the field.
980 #[must_use]
981 pub fn encryption(
982 mut self,
983 encryptor: std::sync::Arc<dyn magnetar_runtime_tokio::MessageEncryptor>,
984 ) -> Self {
985 self.encryptor = Some(encryptor);
986 self
987 }
988
989 /// Open every per-partition producer honoring the configured PIP-4
990 /// encryptor. Tokio-engine-only — use [`Self::create`] for the
991 /// engine-generic path that ignores the encryptor field.
992 ///
993 /// # Errors
994 ///
995 /// - [`PulsarError::Client`] on broker metadata lookup or per-partition open failure.
996 pub async fn create_with_encryption(self) -> Result<PartitionedProducer, PulsarError> {
997 let mut deadline = self.client.runtime_client().operation_timer();
998 let mut last_broker_error = None;
999 let partitions_count = self
1000 .client
1001 .runtime_client()
1002 .partitioned_topic_metadata_with_operation_deadline(
1003 &self.topic,
1004 deadline.as_mut(),
1005 &mut last_broker_error,
1006 )
1007 .await?;
1008
1009 let partition_topics: Vec<String> = if partitions_count == 0 {
1010 vec![self.topic.clone()]
1011 } else {
1012 (0..partitions_count)
1013 .map(|i| format!("{}-partition-{}", self.topic, i))
1014 .collect()
1015 };
1016
1017 let mut child_producers: Vec<Producer> = Vec::with_capacity(partition_topics.len());
1018 for child_topic in &partition_topics {
1019 let req = CreateProducerRequest {
1020 topic: child_topic.clone(),
1021 producer_name: self.name.clone(),
1022 compression: self.compression,
1023 enable_batching: self.enable_batching,
1024 enable_chunking: self.enable_chunking,
1025 max_batch_size_bytes: self.max_batch_size_bytes,
1026 max_messages_in_batch: self.max_messages_in_batch,
1027 schema: self.schema.clone(),
1028 initial_sequence_id: self.initial_sequence_id,
1029 access_mode: self.access_mode,
1030 producer_metadata: self.producer_metadata.clone(),
1031 send_timeout: self.send_timeout,
1032 batching_max_publish_delay: self.batching_max_publish_delay,
1033 };
1034 let result = self
1035 .client
1036 .runtime_client()
1037 .open_producer_with_operation_deadline(
1038 req,
1039 self.encryptor.clone(),
1040 deadline.as_mut(),
1041 &mut last_broker_error,
1042 )
1043 .await;
1044 match result {
1045 Ok(p) => child_producers.push(p),
1046 Err(e) => {
1047 for p in child_producers {
1048 let _ = p.close().await;
1049 }
1050 return Err(PulsarError::Client(e));
1051 }
1052 }
1053 }
1054
1055 let auto_update = self
1056 .auto_update_partitions_interval
1057 .map(|interval| spawn_auto_update_task(self.topic.clone(), interval, partitions_count));
1058
1059 Ok(PartitionedProducer {
1060 partitions: child_producers,
1061 base_topic: self.topic,
1062 routing: self.routing,
1063 router: self.router,
1064 cursor: AtomicU64::new(0),
1065 auto_update,
1066 })
1067 }
1068}
1069
1070// helper to avoid unused-import warning if Bytes isn't needed here
1071#[allow(dead_code)]
1072fn _bytes_in_use() -> Bytes {
1073 Bytes::new()
1074}
1075
1076#[cfg(test)]
1077mod tests {
1078 use super::*;
1079
1080 #[test]
1081 fn key_hash_is_deterministic_and_round_robin_advances() {
1082 let pp: PartitionedProducer = PartitionedProducer {
1083 partitions: Vec::new(),
1084 base_topic: "t".into(),
1085 routing: MessageRoutingMode::KeyHashOrRoundRobin,
1086 router: None,
1087 cursor: AtomicU64::new(0),
1088 auto_update: None,
1089 };
1090 // We can't actually run pick_partition with 0 partitions; emulate by mirroring
1091 // the same `java_string_hash` math the production path uses. Confirms that
1092 // the same key yields the same partition for a given total — both sides
1093 // call into `java_string_hash`.
1094 let pick_a = (java_string_hash("alpha") as usize) % 4;
1095 let pick_b = (java_string_hash("alpha") as usize) % 4;
1096 assert_eq!(pick_a, pick_b);
1097 let _ = pp; // suppress unused
1098 }
1099
1100 /// F3 — `MessageRoutingMode::KeyHashOrRoundRobin` must use Java's
1101 /// `String.hashCode()` (a.k.a. `HashingScheme.JavaStringHash`), not
1102 /// Rust's `DefaultHasher`. Otherwise a Rust-side keyed producer
1103 /// routes the same key to a different partition than a Java
1104 /// producer (or even than a different process — Rust's
1105 /// `DefaultHasher` carries a per-process random seed). This test
1106 /// pins the routing math to the same `java_string_hash` invariants
1107 /// that the existing `HashTest` vectors lock in.
1108 ///
1109 /// Java `"alpha".hashCode()` is `92909918` — we mirror the
1110 /// `& Integer.MAX_VALUE` mask, then `% partitions`. `"abc"` is the
1111 /// textbook value `96354`. Empty key drops to round-robin (and so
1112 /// doesn't enter this branch).
1113 #[test]
1114 fn default_routing_uses_java_string_hash_not_default_hasher() {
1115 // Sanity: the helper still matches the well-known Java values.
1116 assert_eq!(java_string_hash("alpha"), 92_909_918);
1117 assert_eq!(java_string_hash("abc"), 96_354);
1118
1119 // The router branch the production code takes for non-empty
1120 // keys: `(java_string_hash(k) as usize) % n`. We mirror the
1121 // computation directly because constructing a `PartitionedProducer`
1122 // with N>0 partitions requires real per-partition producers
1123 // (which require a broker). The point is parity — same key,
1124 // same partition, regardless of how many times we call it,
1125 // and across processes.
1126 for &(key, partitions, expected_alpha) in &[
1127 ("alpha", 4_usize, (92_909_918_usize) % 4),
1128 ("alpha", 16_usize, (92_909_918_usize) % 16),
1129 ("abc", 8_usize, (96_354_usize) % 8),
1130 ("keykeykeykeykey1", 8_usize, (434_058_482_usize) % 8),
1131 ("keykeykey2", 32_usize, (42_978_643_usize) % 32),
1132 ] {
1133 let pick = (java_string_hash(key) as usize) % partitions;
1134 assert_eq!(
1135 pick, expected_alpha,
1136 "key={key:?} partitions={partitions} must match Java's \
1137 String.hashCode() & Integer.MAX_VALUE then % partitions"
1138 );
1139 }
1140
1141 // Determinism across calls — the previous DefaultHasher path
1142 // failed this within a single process whenever RandomState
1143 // rotated; across processes it failed every restart.
1144 let snapshot_a = (java_string_hash("user-42") as usize) % 16;
1145 let snapshot_b = (java_string_hash("user-42") as usize) % 16;
1146 assert_eq!(snapshot_a, snapshot_b);
1147
1148 // Multi-byte / non-ASCII key — UTF-16 path. Java treats
1149 // non-BMP code points as surrogate pairs, which
1150 // `java_string_hash` mirrors via `encode_utf16`. Use a BMP
1151 // multibyte character so the test is portable.
1152 let multi = "éclair"; // 'é' = U+00E9, 1 UTF-16 unit
1153 let _ = (java_string_hash(multi) as usize) % 8;
1154 // Determinism on multi-byte input.
1155 let a = (java_string_hash(multi) as usize) % 8;
1156 let b = (java_string_hash(multi) as usize) % 8;
1157 assert_eq!(a, b);
1158 }
1159
1160 #[derive(Debug)]
1161 struct ConstantRouter(usize);
1162 impl MessageRouter for ConstantRouter {
1163 fn route(&self, _msg: &OutgoingMessage, _partitions: usize) -> usize {
1164 self.0
1165 }
1166 }
1167
1168 #[test]
1169 fn custom_router_overrides_mode_and_clamps_out_of_range() {
1170 // Build a fake producer with 4 dummy partition slots so pick_partition has range.
1171 // We can't construct real Producers here (needs a broker connection); since we
1172 // only exercise pick_partition's branch logic, give it an empty slice and a
1173 // router that returns a stable value via shimming.
1174 // Instead, exercise the math directly: cap = `idx.min(n - 1)`.
1175 let n = 4_usize;
1176 let idx = 2_usize;
1177 assert_eq!(idx.min(n - 1), 2);
1178 let oor = 999_usize;
1179 assert_eq!(
1180 oor.min(n - 1),
1181 3,
1182 "out-of-range router result clamps to n-1"
1183 );
1184
1185 // Smoke test the trait dispatch path (no producer needed).
1186 let r: std::sync::Arc<dyn MessageRouter> = std::sync::Arc::new(ConstantRouter(2));
1187 let msg = OutgoingMessage::default();
1188 assert_eq!(r.route(&msg, 4), 2);
1189 }
1190
1191 // -- Murmur3 parity with Apache Pulsar's `HashTest.murmur3_32HashTest`. -----------
1192 //
1193 // Vectors copied verbatim from
1194 // `pulsar-client/src/test/java/org/apache/pulsar/client/impl/HashTest.java`. They
1195 // are also the C++ client's expected outputs, so a regression here breaks
1196 // cross-language partition affinity.
1197 #[test]
1198 fn murmur3_matches_java_hashtest_vectors() {
1199 assert_eq!(murmur3_32_hash(b"k1"), 2_110_152_746);
1200 assert_eq!(murmur3_32_hash(b"k2"), 1_479_966_664);
1201 assert_eq!(murmur3_32_hash(b"key1"), 462_881_061);
1202 assert_eq!(murmur3_32_hash(b"key2"), 1_936_800_180);
1203 assert_eq!(murmur3_32_hash(b"key01"), 39_696_932);
1204 assert_eq!(murmur3_32_hash(b"key02"), 751_761_803);
1205 }
1206
1207 #[test]
1208 fn murmur3_handles_empty_input() {
1209 // Empty input under seed=0 with the masking we apply should be 0 (matches
1210 // Java's `Murmur3_32Hash.makeHash(new byte[0]) & Integer.MAX_VALUE`).
1211 assert_eq!(murmur3_32_hash(b""), 0);
1212 }
1213
1214 // -- JavaStringHash parity with Apache Pulsar's `HashTest.javaStringHashTest`. ----
1215 //
1216 // The `"keykeykey2"` value overflows i32 as unsigned (Java's `hashCode()` returns
1217 // negative) — the mask with `Integer.MAX_VALUE` restores the non-negative form.
1218 #[test]
1219 fn java_string_hash_matches_java_hashtest_vectors() {
1220 assert_eq!(java_string_hash("keykeykeykeykey1"), 434_058_482);
1221 assert_eq!(java_string_hash("keykeykey2"), 42_978_643);
1222 // Well-known textbook value: "abc".hashCode() == 96354.
1223 assert_eq!(java_string_hash("abc"), 96_354);
1224 }
1225
1226 #[test]
1227 fn java_string_hash_empty_is_zero() {
1228 // `"".hashCode()` is 0 in Java, masked stays 0.
1229 assert_eq!(java_string_hash(""), 0);
1230 }
1231
1232 // -- Routing determinism: same key always routes to the same partition. ----------
1233 #[test]
1234 fn murmur3_router_is_keyed_and_deterministic() {
1235 let router = Murmur3HashHasher;
1236 let msg = OutgoingMessage::default().key("user-42");
1237 let p0 = router.route(&msg, 16);
1238 // Call ten times — must be stable.
1239 for _ in 0..10 {
1240 assert_eq!(router.route(&msg, 16), p0);
1241 }
1242 // And different keys can land on different partitions (smoke check).
1243 let other = OutgoingMessage::default().key("user-9999");
1244 let p1 = router.route(&other, 16);
1245 // Not asserting `p0 != p1` because hash collisions exist for 16 partitions; we
1246 // just want to prove the value is in range.
1247 assert!(p0 < 16);
1248 assert!(p1 < 16);
1249 }
1250
1251 #[test]
1252 fn java_string_hash_router_is_keyed_and_deterministic() {
1253 let router = JavaStringHashHasher;
1254 let msg = OutgoingMessage::default().key("orders-tenant-A");
1255 let p0 = router.route(&msg, 8);
1256 for _ in 0..10 {
1257 assert_eq!(router.route(&msg, 8), p0);
1258 }
1259 assert!(p0 < 8);
1260 }
1261
1262 // Same key value must land on the same partition under both hashers across
1263 // independent invocations — guards against accidental cursor / RNG bleed-in.
1264 #[test]
1265 fn hashers_have_no_hidden_state() {
1266 let m1 = Murmur3HashHasher;
1267 let m2 = Murmur3HashHasher;
1268 let key = OutgoingMessage::default().key("k1");
1269 assert_eq!(m1.route(&key, 32), m2.route(&key, 32));
1270
1271 let j1 = JavaStringHashHasher;
1272 let j2 = JavaStringHashHasher;
1273 assert_eq!(j1.route(&key, 32), j2.route(&key, 32));
1274 }
1275
1276 // Cross-check Murmur3 routing against the Java expected value mod partition count
1277 // (uses the vector from `HashTest`): "key1" -> 462881061 -> 462881061 % 16 = 5.
1278 #[test]
1279 fn murmur3_router_matches_java_modulo() {
1280 let router = Murmur3HashHasher;
1281 let msg = OutgoingMessage::default().key("key1");
1282 assert_eq!(router.route(&msg, 16), (462_881_061_usize) % 16);
1283 }
1284
1285 // No-key fallback uses sticky partition 0 (router overrides MessageRoutingMode).
1286 #[test]
1287 fn hashers_fall_back_to_partition_zero_without_key() {
1288 let m = Murmur3HashHasher;
1289 let j = JavaStringHashHasher;
1290 let msg = OutgoingMessage::default();
1291 assert_eq!(m.route(&msg, 8), 0);
1292 assert_eq!(j.route(&msg, 8), 0);
1293 let msg_empty = OutgoingMessage::default().key("");
1294 assert_eq!(m.route(&msg_empty, 8), 0);
1295 assert_eq!(j.route(&msg_empty, 8), 0);
1296 }
1297}