Skip to main content

walrq_client/
lib.rs

1use std::collections::HashMap;
2use std::sync::Arc;
3use std::time::Duration;
4use bytes::Bytes;
5use thiserror::Error;
6use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf};
7use tokio::net::TcpStream;
8use tokio::sync::{mpsc, oneshot, Mutex, RwLock};
9use ulid::Ulid;
10
11pub use walrq::protocol::{
12    AckItem, BatchPushItem, Message, Request, Response,
13};
14use walrq::protocol::codec;
15
16#[derive(Debug, Error)]
17pub enum ClientError {
18    #[error("IO error: {0}")]
19    Io(#[from] std::io::Error),
20    #[error("Postcard error: {0}")]
21    Postcard(#[from] postcard::Error),
22    #[error("Server error: {0}")]
23    ServerError(String),
24    #[error("All seed nodes unreachable or max redirects exceeded")]
25    Unavailable,
26    #[error("Invalid redirect target: {0}")]
27    InvalidTarget(String),
28    #[error("Buffer channel closed")]
29    ChannelClosed,
30}
31
32#[derive(Debug, Clone)]
33pub struct ClientConfig {
34    pub buffer_window_ms: u64, // Default: 300ms
35    pub max_batch_size: usize,  // Default: 500
36    pub max_redirects: usize,   // Default: 10
37}
38
39impl Default for ClientConfig {
40    fn default() -> Self {
41        Self {
42            buffer_window_ms: 300,
43            max_batch_size: 500,
44            max_redirects: 10,
45        }
46    }
47}
48
49struct BufferedPushItem {
50    queue: String,
51    message_id: Option<String>,
52    payload: Bytes,
53    delay_seconds: u64,
54    tx: oneshot::Sender<Result<String, ClientError>>,
55}
56
57struct PooledConnection {
58    writer: OwnedWriteHalf,
59    reader: OwnedReadHalf,
60    read_buf: Vec<u8>,
61}
62
63#[derive(Clone)]
64pub struct WalrClient {
65    inner: Arc<ClientInner>,
66    sharded_senders: Vec<mpsc::UnboundedSender<BufferedPushItem>>,
67}
68
69struct ClientInner {
70    seed_nodes: Arc<RwLock<Vec<String>>>,
71    routes: Arc<RwLock<HashMap<String, String>>>,
72    connections: Arc<RwLock<HashMap<String, Vec<Arc<Mutex<PooledConnection>>>>>>,
73    config: ClientConfig,
74    conn_counter: std::sync::atomic::AtomicUsize,
75}
76
77impl WalrClient {
78    pub fn new(seed_nodes: Vec<String>) -> Self {
79        Self::with_config(seed_nodes, ClientConfig::default())
80    }
81
82    pub fn with_config(seed_nodes: Vec<String>, config: ClientConfig) -> Self {
83        let inner = Arc::new(ClientInner {
84            seed_nodes: Arc::new(RwLock::new(seed_nodes)),
85            routes: Arc::new(RwLock::new(HashMap::new())),
86            connections: Arc::new(RwLock::new(HashMap::new())),
87            config: config.clone(),
88            conn_counter: std::sync::atomic::AtomicUsize::new(0),
89        });
90
91        const NUM_SHARDS: usize = 32;
92        let mut sharded_senders = Vec::with_capacity(NUM_SHARDS);
93
94        for _ in 0..NUM_SHARDS {
95            let (tx, mut rx) = mpsc::unbounded_channel::<BufferedPushItem>();
96            sharded_senders.push(tx);
97
98            let inner_clone = Arc::clone(&inner);
99            let window_ms = config.buffer_window_ms;
100            let max_batch = config.max_batch_size;
101
102            tokio::spawn(async move {
103                let mut queues_map: HashMap<String, Vec<(Option<String>, Bytes, u64, oneshot::Sender<Result<String, ClientError>>)>> = HashMap::new();
104                let mut interval = tokio::time::interval_at(
105                    tokio::time::Instant::now() + Duration::from_millis(window_ms),
106                    Duration::from_millis(window_ms),
107                );
108
109                loop {
110                    tokio::select! {
111                        Some(item) = rx.recv() => {
112                            let list = queues_map.entry(item.queue.clone()).or_insert_with(Vec::new);
113                            list.push((item.message_id, item.payload, item.delay_seconds, item.tx));
114
115                            if list.len() >= max_batch {
116                                let items = queues_map.remove(&item.queue).unwrap();
117                                let inner_ref = Arc::clone(&inner_clone);
118                                let q_name = item.queue.clone();
119                                tokio::spawn(async move {
120                                    Self::flush_queue_batch(&inner_ref, &q_name, items).await;
121                                });
122                            }
123                        }
124                        _ = interval.tick() => {
125                            if !queues_map.is_empty() {
126                                let drained: HashMap<_, _> = queues_map.drain().collect();
127                                for (q_name, items) in drained {
128                                    if !items.is_empty() {
129                                        let inner_ref = Arc::clone(&inner_clone);
130                                        tokio::spawn(async move {
131                                            Self::flush_queue_batch(&inner_ref, &q_name, items).await;
132                                        });
133                                    }
134                                }
135                            }
136                        }
137                    }
138                }
139            });
140        }
141
142        Self {
143            inner,
144            sharded_senders,
145        }
146    }
147
148    #[inline(always)]
149    fn compress_wire_payload(payload: &[u8]) -> Vec<u8> {
150        if payload.len() > 1024 {
151            if let Ok(comp) = zstd::encode_all(payload, 1) {
152                if comp.len() + 1 < payload.len() {
153                    let mut out = Vec::with_capacity(comp.len() + 1);
154                    out.push(0x01); // 0x01 = zstd compressed by client
155                    out.extend_from_slice(&comp);
156                    return out;
157                }
158            }
159        }
160        let mut out = Vec::with_capacity(payload.len() + 1);
161        out.push(0x00); // 0x00 = uncompressed
162        out.extend_from_slice(payload);
163        out
164    }
165
166    #[inline(always)]
167    fn decompress_wire_payload(raw: Vec<u8>) -> Vec<u8> {
168        if raw.is_empty() {
169            return raw;
170        }
171        if raw[0] == 0x01 {
172            if let Ok(decomp) = zstd::decode_all(&raw[1..]) {
173                return decomp;
174            }
175        }
176        raw[1..].to_vec()
177    }
178
179    pub async fn update_seeds(&self, seeds: Vec<String>) {
180        let mut s = self.inner.seed_nodes.write().await;
181        *s = seeds;
182    }
183
184    pub async fn push(&self, queue: &str, payload: Bytes, delay_seconds: u64) -> Result<String, ClientError> {
185        self.push_opt(queue, payload, delay_seconds, None).await
186    }
187
188    pub async fn push_with_id(&self, queue: &str, payload: Bytes, delay_seconds: u64, message_id: &str) -> Result<String, ClientError> {
189        self.push_opt(queue, payload, delay_seconds, Some(message_id.to_string())).await
190    }
191
192    pub async fn push_opt(&self, queue: &str, payload: Bytes, delay_seconds: u64, message_id: Option<String>) -> Result<String, ClientError> {
193        let (tx, rx) = oneshot::channel();
194        let bytes = queue.as_bytes();
195        let mut hash = 0xcbf29ce484222325u64;
196        for &b in bytes {
197            hash ^= b as u64;
198            hash = hash.wrapping_mul(0x100000001b3);
199        }
200        let shard = (hash as usize) % self.sharded_senders.len();
201
202        let mid = Some(message_id.unwrap_or_else(|| Ulid::new().to_string()));
203
204        self.sharded_senders[shard]
205            .send(BufferedPushItem {
206                queue: queue.to_string(),
207                message_id: mid,
208                payload,
209                delay_seconds,
210                tx,
211            })
212            .map_err(|_| ClientError::ChannelClosed)?;
213
214        rx.await.map_err(|_| ClientError::ChannelClosed)?
215    }
216
217    pub async fn push_immediate(&self, queue: &str, payload: Bytes, delay_seconds: u64) -> Result<String, ClientError> {
218        self.push_immediate_opt(queue, payload, delay_seconds, None).await
219    }
220
221    pub async fn push_immediate_with_id(&self, queue: &str, payload: Bytes, delay_seconds: u64, message_id: &str) -> Result<String, ClientError> {
222        self.push_immediate_opt(queue, payload, delay_seconds, Some(message_id.to_string())).await
223    }
224
225    pub async fn push_immediate_opt(&self, queue: &str, payload: Bytes, delay_seconds: u64, message_id: Option<String>) -> Result<String, ClientError> {
226        let message_id = message_id.unwrap_or_else(|| Ulid::new().to_string());
227        let req = Request::Push {
228            queue_name: queue.to_string(),
229            payload: Self::compress_wire_payload(&payload),
230            delay_seconds,
231            message_id,
232        };
233
234        match self.inner.send_with_redirect(queue, req).await? {
235            Response::Push { message_id } => Ok(message_id),
236            Response::Error { message } => Err(ClientError::ServerError(message)),
237            other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
238        }
239    }
240
241    pub async fn push_batch(&self, queue: &str, payloads: Vec<Bytes>) -> Result<Vec<String>, ClientError> {
242        let items: Vec<BatchPushItem> = payloads
243            .into_iter()
244            .map(|p| BatchPushItem {
245                payload: Self::compress_wire_payload(&p),
246                delay_seconds: 0,
247                message_id: Ulid::new().to_string(),
248            })
249            .collect();
250        self.push_batch_internal(queue, items).await
251    }
252
253    pub async fn push_batch_with_delay(&self, queue: &str, payloads: Vec<Bytes>, delay_seconds: u64) -> Result<Vec<String>, ClientError> {
254        let items: Vec<BatchPushItem> = payloads
255            .into_iter()
256            .map(|p| BatchPushItem {
257                payload: Self::compress_wire_payload(&p),
258                delay_seconds,
259                message_id: Ulid::new().to_string(),
260            })
261            .collect();
262        self.push_batch_internal(queue, items).await
263    }
264
265    pub async fn push_batch_with_ids(&self, queue: &str, items: Vec<(Bytes, Option<String>)>) -> Result<Vec<String>, ClientError> {
266        let proto_items: Vec<BatchPushItem> = items
267            .into_iter()
268            .map(|(p, id)| BatchPushItem {
269                payload: Self::compress_wire_payload(&p),
270                delay_seconds: 0,
271                message_id: id.unwrap_or_else(|| Ulid::new().to_string()),
272            })
273            .collect();
274        self.push_batch_internal(queue, proto_items).await
275    }
276
277    async fn push_batch_internal(&self, queue: &str, items: Vec<BatchPushItem>) -> Result<Vec<String>, ClientError> {
278        let req = Request::PushBatch {
279            queue_name: queue.to_string(),
280            items,
281        };
282
283        match self.inner.send_with_redirect(queue, req).await? {
284            Response::PushBatch { message_ids } => Ok(message_ids),
285            Response::Error { message } => Err(ClientError::ServerError(message)),
286            other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
287        }
288    }
289
290    async fn flush_queue_batch(
291        inner: &Arc<ClientInner>,
292        queue: &str,
293        items: Vec<(Option<String>, Bytes, u64, oneshot::Sender<Result<String, ClientError>>)>,
294    ) {
295        let mut senders = Vec::with_capacity(items.len());
296        let mut proto_items = Vec::with_capacity(items.len());
297
298        for (mid, payload, delay_seconds, tx) in items {
299            senders.push(tx);
300            proto_items.push(BatchPushItem {
301                payload: Self::compress_wire_payload(&payload),
302                delay_seconds,
303                message_id: mid.unwrap_or_else(|| Ulid::new().to_string()),
304            });
305        }
306
307        let req = Request::PushBatch {
308            queue_name: queue.to_string(),
309            items: proto_items,
310        };
311
312        match inner.send_with_redirect(queue, req).await {
313            Ok(Response::PushBatch { message_ids }) => {
314                for (tx, id) in senders.into_iter().zip(message_ids.into_iter()) {
315                    let _ = tx.send(Ok(id));
316                }
317            }
318            Ok(Response::Error { message }) => {
319                for tx in senders {
320                    let _ = tx.send(Err(ClientError::ServerError(message.clone())));
321                }
322            }
323            _ => {
324                for tx in senders {
325                    let _ = tx.send(Err(ClientError::Unavailable));
326                }
327            }
328        }
329    }
330
331    pub async fn poll(&self, queue: &str, visibility_timeout_sec: u32, batch_size: u32) -> Result<Vec<Message>, ClientError> {
332        let req = Request::Poll {
333            queue_name: queue.to_string(),
334            visibility_timeout_sec,
335            batch_size: if batch_size == 0 { 1 } else { batch_size },
336        };
337
338        match self.inner.send_with_redirect(queue, req).await? {
339            Response::Poll { messages } => {
340                let decompressed = messages.into_iter().map(|mut m| {
341                    m.payload = Self::decompress_wire_payload(m.payload);
342                    m
343                }).collect();
344                Ok(decompressed)
345            }
346            Response::Error { message } => Err(ClientError::ServerError(message)),
347            other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
348        }
349    }
350
351    pub async fn ack(&self, queue: &str, message_id: &str, receipt_handle: &str) -> Result<bool, ClientError> {
352        let req = Request::Ack {
353            queue_name: queue.to_string(),
354            message_id: message_id.to_string(),
355            receipt_handle: receipt_handle.to_string(),
356        };
357
358        match self.inner.send_with_redirect(queue, req).await? {
359            Response::Ack { success } => Ok(success),
360            Response::Error { message } => Err(ClientError::ServerError(message)),
361            other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
362        }
363    }
364
365    pub async fn ack_batch(&self, queue: &str, items: Vec<(String, String)>) -> Result<u32, ClientError> {
366        let ack_items: Vec<AckItem> = items
367            .into_iter()
368            .map(|(message_id, receipt_handle)| AckItem {
369                message_id,
370                receipt_handle,
371            })
372            .collect();
373
374        self.ack_batch_items(queue, ack_items).await
375    }
376
377    pub async fn ack_batch_items(&self, queue: &str, items: Vec<AckItem>) -> Result<u32, ClientError> {
378        let req = Request::AckBatch {
379            queue_name: queue.to_string(),
380            items,
381        };
382
383        match self.inner.send_with_redirect(queue, req).await? {
384            Response::AckBatch { acked_count } => Ok(acked_count),
385            Response::Error { message } => Err(ClientError::ServerError(message)),
386            other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
387        }
388    }
389
390    /// Fetch Prometheus text format metrics directly from cluster
391    pub async fn get_metrics(&self) -> Result<String, ClientError> {
392        let req = Request::Metrics;
393        match self.inner.send_with_redirect("", req).await? {
394            Response::Metrics { prometheus_text } => Ok(prometheus_text),
395            Response::Error { message } => Err(ClientError::ServerError(message)),
396            other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
397        }
398    }
399
400    /// Dynamically add a new node to the Raft cluster
401    pub async fn join_cluster(&self, new_peer_addr: &str) -> Result<Vec<String>, ClientError> {
402        let req = Request::JoinCluster {
403            peer_addr: new_peer_addr.to_string(),
404        };
405        match self.inner.send_with_redirect("", req).await? {
406            Response::ClusterMembership { success: true, members } => Ok(members),
407            Response::Error { message } => Err(ClientError::ServerError(message)),
408            other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
409        }
410    }
411
412    /// Dynamically remove an existing node from the Raft cluster
413    pub async fn leave_cluster(&self, peer_addr: &str) -> Result<Vec<String>, ClientError> {
414        let req = Request::LeaveCluster {
415            peer_addr: peer_addr.to_string(),
416        };
417        match self.inner.send_with_redirect("", req).await? {
418            Response::ClusterMembership { success: true, members } => Ok(members),
419            Response::Error { message } => Err(ClientError::ServerError(message)),
420            other => Err(ClientError::ServerError(format!("Unexpected response: {:?}", other))),
421        }
422    }
423}
424
425impl ClientInner {
426    async fn resolve_initial_target(&self, queue: &str) -> Result<String, ClientError> {
427        {
428            let routes = self.routes.read().await;
429            if let Some(target) = routes.get(queue) {
430                return Ok(target.clone());
431            }
432        }
433
434        let seeds = self.seed_nodes.read().await;
435        if seeds.is_empty() {
436            return Err(ClientError::Unavailable);
437        }
438
439        let bytes = queue.as_bytes();
440        let mut hash = 0xcbf29ce484222325u64;
441        for &b in bytes {
442            hash ^= b as u64;
443            hash = hash.wrapping_mul(0x100000001b3);
444        }
445        let idx = (hash as usize) % seeds.len();
446        Ok(seeds[idx].clone())
447    }
448
449    async fn update_route(&self, queue: &str, target: &str) {
450        let mut routes = self.routes.write().await;
451        routes.insert(queue.to_string(), target.to_string());
452    }
453
454    async fn invalidate_route(&self, queue: &str, target: &str) {
455        let mut routes = self.routes.write().await;
456        if let Some(current) = routes.get(queue) {
457            if current == target {
458                routes.remove(queue);
459            }
460        }
461    }
462
463    async fn get_connection(&self, target: &str) -> Result<Arc<Mutex<PooledConnection>>, ClientError> {
464        const POOL_SIZE_PER_HOST: usize = 4;
465
466        {
467            let conns = self.connections.read().await;
468            if let Some(pool) = conns.get(target) {
469                if !pool.is_empty() {
470                    let idx = self.conn_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % pool.len();
471                    return Ok(Arc::clone(&pool[idx]));
472                }
473            }
474        }
475
476        let mut conns = self.connections.write().await;
477        let pool = conns.entry(target.to_string()).or_insert_with(Vec::new);
478
479        if pool.len() < POOL_SIZE_PER_HOST {
480            let stream = TcpStream::connect(target).await?;
481            let _ = stream.set_nodelay(true);
482            let (reader, writer) = stream.into_split();
483            let handle = Arc::new(Mutex::new(PooledConnection {
484                writer,
485                reader,
486                read_buf: Vec::with_capacity(16 * 1024),
487            }));
488            pool.push(Arc::clone(&handle));
489            Ok(handle)
490        } else {
491            let idx = self.conn_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % pool.len();
492            Ok(Arc::clone(&pool[idx]))
493        }
494    }
495
496    async fn remove_connection(&self, target: &str) {
497        let mut conns = self.connections.write().await;
498        conns.remove(target);
499    }
500
501    async fn send_with_redirect(&self, queue: &str, req: Request) -> Result<Response, ClientError> {
502        let mut target = self.resolve_initial_target(queue).await?;
503        let mut attempts = 0;
504
505        while attempts <= self.config.max_redirects {
506            let conn_handle = match self.get_connection(&target).await {
507                Ok(c) => c,
508                Err(_) => {
509                    self.invalidate_route(queue, &target).await;
510                    self.remove_connection(&target).await;
511                    let seeds = self.seed_nodes.read().await;
512                    if let Some(next_seed) = seeds.iter().find(|&s| s != &target) {
513                        target = next_seed.clone();
514                        attempts += 1;
515                        continue;
516                    }
517                    return Err(ClientError::Unavailable);
518                }
519            };
520
521            let mut conn = conn_handle.lock().await;
522            let PooledConnection { ref mut reader, ref mut writer, ref mut read_buf } = *conn;
523            if codec::write_message(writer, &req).await.is_err() {
524                drop(conn);
525                self.invalidate_route(queue, &target).await;
526                self.remove_connection(&target).await;
527                attempts += 1;
528                continue;
529            }
530
531            match codec::read_message_with_buf::<_, Response>(reader, read_buf).await {
532                Ok(Response::Redirect { leader }) => {
533                    drop(conn);
534                    if !leader.is_empty() {
535                        target = leader;
536                    } else {
537                        let seeds = self.seed_nodes.read().await;
538                        if let Some(next_seed) = seeds.iter().find(|&s| s != &target) {
539                            target = next_seed.clone();
540                        }
541                    }
542                    attempts += 1;
543                    continue;
544                }
545                Ok(resp) => {
546                    self.update_route(queue, &target).await;
547                    return Ok(resp);
548                }
549                Err(_) => {
550                    drop(conn);
551                    self.invalidate_route(queue, &target).await;
552                    self.remove_connection(&target).await;
553                    attempts += 1;
554                }
555            }
556        }
557
558        Err(ClientError::Unavailable)
559    }
560}