1use std::collections::HashMap;
2use std::sync::Arc;
3use std::time::Duration;
4use bytes::Bytes;
5use thiserror::Error;
6use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf};
7use tokio::net::TcpStream;
8use tokio::sync::{mpsc, oneshot, Mutex, RwLock};
9use ulid::Ulid;
10
11pub use walrq::protocol::{
12 AckItem, BatchPushItem, Message, Request, Response,
13};
14use walrq::protocol::codec;
15
16#[derive(Debug, Error)]
17pub enum ClientError {
18 #[error("IO error: {0}")]
19 Io(#[from] std::io::Error),
20 #[error("Postcard error: {0}")]
21 Postcard(#[from] postcard::Error),
22 #[error("Server error: {0}")]
23 ServerError(String),
24 #[error("All seed nodes unreachable or max redirects exceeded")]
25 Unavailable,
26 #[error("Invalid redirect target: {0}")]
27 InvalidTarget(String),
28 #[error("Buffer channel closed")]
29 ChannelClosed,
30}
31
32#[derive(Debug, Clone)]
33pub struct ClientConfig {
34 pub buffer_window_ms: u64, pub max_batch_size: usize, pub max_redirects: usize, }
38
39impl Default for ClientConfig {
40 fn default() -> Self {
41 Self {
42 buffer_window_ms: 300,
43 max_batch_size: 500,
44 max_redirects: 10,
45 }
46 }
47}
48
49struct BufferedPushItem {
50 queue: String,
51 message_id: Option<String>,
52 payload: Bytes,
53 delay_seconds: u64,
54 tx: oneshot::Sender<Result<String, ClientError>>,
55}
56
57struct PooledConnection {
58 writer: OwnedWriteHalf,
59 reader: OwnedReadHalf,
60 read_buf: Vec<u8>,
61}
62
63#[derive(Clone)]
64pub struct WalrClient {
65 inner: Arc<ClientInner>,
66 sharded_senders: Vec<mpsc::UnboundedSender<BufferedPushItem>>,
67}
68
69struct ClientInner {
70 seed_nodes: Arc<RwLock<Vec<String>>>,
71 routes: Arc<RwLock<HashMap<String, String>>>,
72 connections: Arc<RwLock<HashMap<String, Vec<Arc<Mutex<PooledConnection>>>>>>,
73 config: ClientConfig,
74 conn_counter: std::sync::atomic::AtomicUsize,
75}
76
77impl WalrClient {
78 pub fn new(seed_nodes: Vec<String>) -> Self {
79 Self::with_config(seed_nodes, ClientConfig::default())
80 }
81
82 pub fn with_config(seed_nodes: Vec<String>, config: ClientConfig) -> Self {
83 let inner = Arc::new(ClientInner {
84 seed_nodes: Arc::new(RwLock::new(seed_nodes)),
85 routes: Arc::new(RwLock::new(HashMap::new())),
86 connections: Arc::new(RwLock::new(HashMap::new())),
87 config: config.clone(),
88 conn_counter: std::sync::atomic::AtomicUsize::new(0),
89 });
90
91 const NUM_SHARDS: usize = 32;
92 let mut sharded_senders = Vec::with_capacity(NUM_SHARDS);
93
94 for _ in 0..NUM_SHARDS {
95 let (tx, mut rx) = mpsc::unbounded_channel::<BufferedPushItem>();
96 sharded_senders.push(tx);
97
98 let inner_clone = Arc::clone(&inner);
99 let window_ms = config.buffer_window_ms;
100 let max_batch = config.max_batch_size;
101
102 tokio::spawn(async move {
103 let mut queues_map: HashMap<String, Vec<(Option<String>, Bytes, u64, oneshot::Sender<Result<String, ClientError>>)>> = HashMap::new();
104 let mut interval = tokio::time::interval_at(
105 tokio::time::Instant::now() + Duration::from_millis(window_ms),
106 Duration::from_millis(window_ms),
107 );
108
109 loop {
110 tokio::select! {
111 Some(item) = rx.recv() => {
112 let list = queues_map.entry(item.queue.clone()).or_insert_with(Vec::new);
113 list.push((item.message_id, item.payload, item.delay_seconds, item.tx));
114
115 if list.len() >= max_batch {
116 let items = queues_map.remove(&item.queue).unwrap();
117 let inner_ref = Arc::clone(&inner_clone);
118 let q_name = item.queue.clone();
119 tokio::spawn(async move {
120 Self::flush_queue_batch(&inner_ref, &q_name, items).await;
121 });
122 }
123 }
124 _ = interval.tick() => {
125 if !queues_map.is_empty() {
126 let drained: HashMap<_, _> = queues_map.drain().collect();
127 for (q_name, items) in drained {
128 if !items.is_empty() {
129 let inner_ref = Arc::clone(&inner_clone);
130 tokio::spawn(async move {
131 Self::flush_queue_batch(&inner_ref, &q_name, items).await;
132 });
133 }
134 }
135 }
136 }
137 }
138 }
139 });
140 }
141
142 Self {
143 inner,
144 sharded_senders,
145 }
146 }
147
148 #[inline(always)]
149 fn compress_wire_payload(payload: &[u8]) -> Vec<u8> {
150 if payload.len() > 1024 {
151 if let Ok(comp) = zstd::encode_all(payload, 1) {
152 if comp.len() + 1 < payload.len() {
153 let mut out = Vec::with_capacity(comp.len() + 1);
154 out.push(0x01); out.extend_from_slice(&comp);
156 return out;
157 }
158 }
159 }
160 let mut out = Vec::with_capacity(payload.len() + 1);
161 out.push(0x00); out.extend_from_slice(payload);
163 out
164 }
165
166 #[inline(always)]
167 fn decompress_wire_payload(raw: Vec<u8>) -> Vec<u8> {
168 if raw.is_empty() {
169 return raw;
170 }
171 if raw[0] == 0x01 {
172 if let Ok(decomp) = zstd::decode_all(&raw[1..]) {
173 return decomp;
174 }
175 }
176 raw[1..].to_vec()
177 }
178
179 pub async fn update_seeds(&self, seeds: Vec<String>) {
180 let mut s = self.inner.seed_nodes.write().await;
181 *s = seeds;
182 }
183
184 pub async fn push(&self, queue: &str, payload: Bytes, delay_seconds: u64) -> Result<String, ClientError> {
185 self.push_opt(queue, payload, delay_seconds, None).await
186 }
187
188 pub async fn push_with_id(&self, queue: &str, payload: Bytes, delay_seconds: u64, message_id: &str) -> Result<String, ClientError> {
189 self.push_opt(queue, payload, delay_seconds, Some(message_id.to_string())).await
190 }
191
192 pub async fn push_opt(&self, queue: &str, payload: Bytes, delay_seconds: u64, message_id: Option<String>) -> Result<String, ClientError> {
193 let (tx, rx) = oneshot::channel();
194 let bytes = queue.as_bytes();
195 let mut hash = 0xcbf29ce484222325u64;
196 for &b in bytes {
197 hash ^= b as u64;
198 hash = hash.wrapping_mul(0x100000001b3);
199 }
200 let shard = (hash as usize) % self.sharded_senders.len();
201
202 let mid = Some(message_id.unwrap_or_else(|| Ulid::new().to_string()));
203
204 self.sharded_senders[shard]
205 .send(BufferedPushItem {
206 queue: queue.to_string(),
207 message_id: mid,
208 payload,
209 delay_seconds,
210 tx,
211 })
212 .map_err(|_| ClientError::ChannelClosed)?;
213
214 rx.await.map_err(|_| ClientError::ChannelClosed)?
215 }
216
217 pub async fn push_immediate(&self, queue: &str, payload: Bytes, delay_seconds: u64) -> Result<String, ClientError> {
218 self.push_immediate_opt(queue, payload, delay_seconds, None).await
219 }
220
221 pub async fn push_immediate_with_id(&self, queue: &str, payload: Bytes, delay_seconds: u64, message_id: &str) -> Result<String, ClientError> {
222 self.push_immediate_opt(queue, payload, delay_seconds, Some(message_id.to_string())).await
223 }
224
225 pub async fn push_immediate_opt(&self, queue: &str, payload: Bytes, delay_seconds: u64, message_id: Option<String>) -> Result<String, ClientError> {
226 let message_id = message_id.unwrap_or_else(|| Ulid::new().to_string());
227 let req = Request::Push {
228 queue_name: queue.to_string(),
229 payload: Self::compress_wire_payload(&payload),
230 delay_seconds,
231 message_id,
232 };
233
234 match self.inner.send_with_redirect(queue, req).await? {
235 Response::Push { message_id } => Ok(message_id),
236 Response::Error { message } => Err(ClientError::ServerError(message)),
237 other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
238 }
239 }
240
241 pub async fn push_batch(&self, queue: &str, payloads: Vec<Bytes>) -> Result<Vec<String>, ClientError> {
242 let items: Vec<BatchPushItem> = payloads
243 .into_iter()
244 .map(|p| BatchPushItem {
245 payload: Self::compress_wire_payload(&p),
246 delay_seconds: 0,
247 message_id: Ulid::new().to_string(),
248 })
249 .collect();
250 self.push_batch_internal(queue, items).await
251 }
252
253 pub async fn push_batch_with_delay(&self, queue: &str, payloads: Vec<Bytes>, delay_seconds: u64) -> Result<Vec<String>, ClientError> {
254 let items: Vec<BatchPushItem> = payloads
255 .into_iter()
256 .map(|p| BatchPushItem {
257 payload: Self::compress_wire_payload(&p),
258 delay_seconds,
259 message_id: Ulid::new().to_string(),
260 })
261 .collect();
262 self.push_batch_internal(queue, items).await
263 }
264
265 pub async fn push_batch_with_ids(&self, queue: &str, items: Vec<(Bytes, Option<String>)>) -> Result<Vec<String>, ClientError> {
266 let proto_items: Vec<BatchPushItem> = items
267 .into_iter()
268 .map(|(p, id)| BatchPushItem {
269 payload: Self::compress_wire_payload(&p),
270 delay_seconds: 0,
271 message_id: id.unwrap_or_else(|| Ulid::new().to_string()),
272 })
273 .collect();
274 self.push_batch_internal(queue, proto_items).await
275 }
276
277 async fn push_batch_internal(&self, queue: &str, items: Vec<BatchPushItem>) -> Result<Vec<String>, ClientError> {
278 let req = Request::PushBatch {
279 queue_name: queue.to_string(),
280 items,
281 };
282
283 match self.inner.send_with_redirect(queue, req).await? {
284 Response::PushBatch { message_ids } => Ok(message_ids),
285 Response::Error { message } => Err(ClientError::ServerError(message)),
286 other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
287 }
288 }
289
290 async fn flush_queue_batch(
291 inner: &Arc<ClientInner>,
292 queue: &str,
293 items: Vec<(Option<String>, Bytes, u64, oneshot::Sender<Result<String, ClientError>>)>,
294 ) {
295 let mut senders = Vec::with_capacity(items.len());
296 let mut proto_items = Vec::with_capacity(items.len());
297
298 for (mid, payload, delay_seconds, tx) in items {
299 senders.push(tx);
300 proto_items.push(BatchPushItem {
301 payload: Self::compress_wire_payload(&payload),
302 delay_seconds,
303 message_id: mid.unwrap_or_else(|| Ulid::new().to_string()),
304 });
305 }
306
307 let req = Request::PushBatch {
308 queue_name: queue.to_string(),
309 items: proto_items,
310 };
311
312 match inner.send_with_redirect(queue, req).await {
313 Ok(Response::PushBatch { message_ids }) => {
314 for (tx, id) in senders.into_iter().zip(message_ids.into_iter()) {
315 let _ = tx.send(Ok(id));
316 }
317 }
318 Ok(Response::Error { message }) => {
319 for tx in senders {
320 let _ = tx.send(Err(ClientError::ServerError(message.clone())));
321 }
322 }
323 _ => {
324 for tx in senders {
325 let _ = tx.send(Err(ClientError::Unavailable));
326 }
327 }
328 }
329 }
330
331 pub async fn poll(&self, queue: &str, visibility_timeout_sec: u32, batch_size: u32) -> Result<Vec<Message>, ClientError> {
332 let req = Request::Poll {
333 queue_name: queue.to_string(),
334 visibility_timeout_sec,
335 batch_size: if batch_size == 0 { 1 } else { batch_size },
336 };
337
338 match self.inner.send_with_redirect(queue, req).await? {
339 Response::Poll { messages } => {
340 let decompressed = messages.into_iter().map(|mut m| {
341 m.payload = Self::decompress_wire_payload(m.payload);
342 m
343 }).collect();
344 Ok(decompressed)
345 }
346 Response::Error { message } => Err(ClientError::ServerError(message)),
347 other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
348 }
349 }
350
351 pub async fn ack(&self, queue: &str, message_id: &str, receipt_handle: &str) -> Result<bool, ClientError> {
352 let req = Request::Ack {
353 queue_name: queue.to_string(),
354 message_id: message_id.to_string(),
355 receipt_handle: receipt_handle.to_string(),
356 };
357
358 match self.inner.send_with_redirect(queue, req).await? {
359 Response::Ack { success } => Ok(success),
360 Response::Error { message } => Err(ClientError::ServerError(message)),
361 other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
362 }
363 }
364
365 pub async fn ack_batch(&self, queue: &str, items: Vec<(String, String)>) -> Result<u32, ClientError> {
366 let ack_items: Vec<AckItem> = items
367 .into_iter()
368 .map(|(message_id, receipt_handle)| AckItem {
369 message_id,
370 receipt_handle,
371 })
372 .collect();
373
374 self.ack_batch_items(queue, ack_items).await
375 }
376
377 pub async fn ack_batch_items(&self, queue: &str, items: Vec<AckItem>) -> Result<u32, ClientError> {
378 let req = Request::AckBatch {
379 queue_name: queue.to_string(),
380 items,
381 };
382
383 match self.inner.send_with_redirect(queue, req).await? {
384 Response::AckBatch { acked_count } => Ok(acked_count),
385 Response::Error { message } => Err(ClientError::ServerError(message)),
386 other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
387 }
388 }
389
390 pub async fn get_metrics(&self) -> Result<String, ClientError> {
392 let req = Request::Metrics;
393 match self.inner.send_with_redirect("", req).await? {
394 Response::Metrics { prometheus_text } => Ok(prometheus_text),
395 Response::Error { message } => Err(ClientError::ServerError(message)),
396 other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
397 }
398 }
399
400 pub async fn join_cluster(&self, new_peer_addr: &str) -> Result<Vec<String>, ClientError> {
402 let req = Request::JoinCluster {
403 peer_addr: new_peer_addr.to_string(),
404 };
405 match self.inner.send_with_redirect("", req).await? {
406 Response::ClusterMembership { success: true, members } => Ok(members),
407 Response::Error { message } => Err(ClientError::ServerError(message)),
408 other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
409 }
410 }
411
412 pub async fn leave_cluster(&self, peer_addr: &str) -> Result<Vec<String>, ClientError> {
414 let req = Request::LeaveCluster {
415 peer_addr: peer_addr.to_string(),
416 };
417 match self.inner.send_with_redirect("", req).await? {
418 Response::ClusterMembership { success: true, members } => Ok(members),
419 Response::Error { message } => Err(ClientError::ServerError(message)),
420 other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
421 }
422 }
423}
424
425impl ClientInner {
426 async fn resolve_initial_target(&self, queue: &str) -> Result<String, ClientError> {
427 {
428 let routes = self.routes.read().await;
429 if let Some(target) = routes.get(queue) {
430 return Ok(target.clone());
431 }
432 }
433
434 let seeds = self.seed_nodes.read().await;
435 if seeds.is_empty() {
436 return Err(ClientError::Unavailable);
437 }
438
439 let bytes = queue.as_bytes();
440 let mut hash = 0xcbf29ce484222325u64;
441 for &b in bytes {
442 hash ^= b as u64;
443 hash = hash.wrapping_mul(0x100000001b3);
444 }
445 let idx = (hash as usize) % seeds.len();
446 Ok(seeds[idx].clone())
447 }
448
449 async fn update_route(&self, queue: &str, target: &str) {
450 let mut routes = self.routes.write().await;
451 routes.insert(queue.to_string(), target.to_string());
452 }
453
454 async fn invalidate_route(&self, queue: &str, target: &str) {
455 let mut routes = self.routes.write().await;
456 if let Some(current) = routes.get(queue) {
457 if current == target {
458 routes.remove(queue);
459 }
460 }
461 }
462
463 async fn get_connection(&self, target: &str) -> Result<Arc<Mutex<PooledConnection>>, ClientError> {
464 const POOL_SIZE_PER_HOST: usize = 4;
465
466 {
467 let conns = self.connections.read().await;
468 if let Some(pool) = conns.get(target) {
469 if !pool.is_empty() {
470 let idx = self.conn_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % pool.len();
471 return Ok(Arc::clone(&pool[idx]));
472 }
473 }
474 }
475
476 let mut conns = self.connections.write().await;
477 let pool = conns.entry(target.to_string()).or_insert_with(Vec::new);
478
479 if pool.len() < POOL_SIZE_PER_HOST {
480 let stream = TcpStream::connect(target).await?;
481 let _ = stream.set_nodelay(true);
482 let (reader, writer) = stream.into_split();
483 let handle = Arc::new(Mutex::new(PooledConnection {
484 writer,
485 reader,
486 read_buf: Vec::with_capacity(16 * 1024),
487 }));
488 pool.push(Arc::clone(&handle));
489 Ok(handle)
490 } else {
491 let idx = self.conn_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % pool.len();
492 Ok(Arc::clone(&pool[idx]))
493 }
494 }
495
496 async fn remove_connection(&self, target: &str) {
497 let mut conns = self.connections.write().await;
498 conns.remove(target);
499 }
500
501 async fn send_with_redirect(&self, queue: &str, req: Request) -> Result<Response, ClientError> {
502 let mut target = self.resolve_initial_target(queue).await?;
503 let mut attempts = 0;
504
505 while attempts <= self.config.max_redirects {
506 let conn_handle = match self.get_connection(&target).await {
507 Ok(c) => c,
508 Err(_) => {
509 self.invalidate_route(queue, &target).await;
510 self.remove_connection(&target).await;
511 let seeds = self.seed_nodes.read().await;
512 if let Some(next_seed) = seeds.iter().find(|&s| s != &target) {
513 target = next_seed.clone();
514 attempts += 1;
515 continue;
516 }
517 return Err(ClientError::Unavailable);
518 }
519 };
520
521 let mut conn = conn_handle.lock().await;
522 let PooledConnection { ref mut reader, ref mut writer, ref mut read_buf } = *conn;
523 if codec::write_message(writer, &req).await.is_err() {
524 drop(conn);
525 self.invalidate_route(queue, &target).await;
526 self.remove_connection(&target).await;
527 attempts += 1;
528 continue;
529 }
530
531 match codec::read_message_with_buf::<_, Response>(reader, read_buf).await {
532 Ok(Response::Redirect { leader }) => {
533 drop(conn);
534 if !leader.is_empty() {
535 target = leader;
536 } else {
537 let seeds = self.seed_nodes.read().await;
538 if let Some(next_seed) = seeds.iter().find(|&s| s != &target) {
539 target = next_seed.clone();
540 }
541 }
542 attempts += 1;
543 continue;
544 }
545 Ok(resp) => {
546 self.update_route(queue, &target).await;
547 return Ok(resp);
548 }
549 Err(_) => {
550 drop(conn);
551 self.invalidate_route(queue, &target).await;
552 self.remove_connection(&target).await;
553 attempts += 1;
554 }
555 }
556 }
557
558 Err(ClientError::Unavailable)
559 }
560}