1use alloc::format;
14use alloc::string::ToString;
15use alloc::sync::Arc;
16use alloc::vec::Vec;
17
18use core::time::Duration;
19
20use std::net::TcpStream;
21use std::thread::JoinHandle;
22
23use liminal::protocol::{Frame, SchemaId, decode};
24use liminal_protocol::outcome::ReconnectState;
25use spin::Mutex;
26
27use crate::SdkError;
28use crate::remote::SETUP_TIMEOUT;
29use crate::remote::mailbox::{
30 Mailbox, MailboxItem, MailboxLimits, MailboxStats, MailboxWords, ReaderEnd, offer_or_end,
31};
32
33const WEBSOCKET_SUBSCRIPTION_WORDS: MailboxWords = MailboxWords {
36 receive: "websocket subscription receive failed",
37 item: "delivery",
38 reader: "subscription reader",
39};
40
41use super::binding::{AttemptFateOutcome, OpenRequestDecision, WebSocketAuthorityBinding};
42use super::connection_error;
43use super::core::{
44 DriverOutput, FrameCorrelation, ResponseExpectation, SocketCommand, SocketEvent,
45 WebSocketFrameDriver,
46};
47use super::liminal_ws_message_bound;
48use super::std_socket::{SocketRead, WsSocket};
49
50const CLIENT_MIN_VERSION: liminal::protocol::ProtocolVersion =
52 liminal::protocol::ProtocolVersion::new(1, 0);
53const CLIENT_MAX_VERSION: liminal::protocol::ProtocolVersion =
56 liminal::protocol::DELIVER_MESSAGE_ID_VERSION;
57const SUBSCRIPTION_STREAM_ID: u32 = 1;
59const SUBSCRIBE_MAX_IN_FLIGHT: u32 = 1024;
61
62#[derive(Clone, Debug, PartialEq, Eq)]
68pub struct WebSocketDeliveredMessage {
69 delivery_seq: u64,
70 schema_id: SchemaId,
71 payload: Vec<u8>,
72 message: Option<liminal::protocol::DeliveredId>,
74}
75
76impl WebSocketDeliveredMessage {
77 #[must_use]
79 pub const fn delivery_seq(&self) -> u64 {
80 self.delivery_seq
81 }
82
83 #[must_use]
85 pub const fn schema_id(&self) -> SchemaId {
86 self.schema_id
87 }
88
89 #[must_use]
91 pub fn payload(&self) -> &[u8] {
92 &self.payload
93 }
94
95 #[must_use]
97 pub fn into_payload(self) -> Vec<u8> {
98 self.payload
99 }
100
101 #[must_use]
105 pub const fn message(&self) -> Option<liminal::protocol::DeliveredId> {
106 self.message
107 }
108
109 #[must_use]
112 pub fn reply_parent(&self) -> Option<liminal::protocol::MessageId> {
113 self.message
114 .map(liminal::protocol::DeliveredId::reply_parent)
115 }
116}
117
118impl MailboxItem for WebSocketDeliveredMessage {
119 fn payload_len(&self) -> usize {
120 self.payload.len()
121 }
122}
123
124#[derive(Debug)]
127pub struct WebSocketSubscriptionStream {
128 shutdown: TcpStream,
130 subscription_id: u64,
131 mailbox: Arc<Mailbox<WebSocketDeliveredMessage>>,
137 binding: Arc<Mutex<WebSocketAuthorityBinding>>,
138 reader: Option<JoinHandle<()>>,
139}
140
141impl WebSocketSubscriptionStream {
142 pub fn open(
154 address: &str,
155 channel: &str,
156 accepted_schemas: Vec<SchemaId>,
157 ) -> Result<Self, SdkError> {
158 Self::open_with_auth(address, channel, accepted_schemas, &[])
159 }
160
161 pub fn open_with_auth(
182 address: &str,
183 channel: &str,
184 accepted_schemas: Vec<SchemaId>,
185 auth_token: &[u8],
186 ) -> Result<Self, SdkError> {
187 Self::open_with_limits(
188 address,
189 channel,
190 accepted_schemas,
191 auth_token,
192 MailboxLimits::SUBSCRIPTION,
193 )
194 }
195
196 fn open_with_limits(
198 address: &str,
199 channel: &str,
200 accepted_schemas: Vec<SchemaId>,
201 auth_token: &[u8],
202 limits: MailboxLimits,
203 ) -> Result<Self, SdkError> {
204 let message_bound = liminal_ws_message_bound()?;
205 let mut binding = WebSocketAuthorityBinding::new();
206 match binding.request_open() {
207 OpenRequestDecision::Authorized { .. } => {}
208 OpenRequestDecision::Refused(refusal) => {
209 return Err(connection_error(&format!(
210 "client authority refused the subscription open: {refusal:?}"
211 )));
212 }
213 }
214 match Self::open_link(
215 address,
216 channel,
217 accepted_schemas,
218 message_bound,
219 auth_token,
220 ) {
221 Ok((socket, driver, subscription_id, pending)) => {
222 match binding.connection_established() {
223 AttemptFateOutcome::Recorded { .. } => {}
224 AttemptFateOutcome::Refused(refusal) => {
225 return Err(SdkError::Protocol {
226 description: format!(
227 "client authority refused the Connected fate for the \
228 subscription open: {refusal:?}"
229 ),
230 });
231 }
232 }
233 Self::start(socket, driver, binding, subscription_id, pending, limits)
234 }
235 Err(error) => match binding.open_failed() {
236 AttemptFateOutcome::Recorded { .. } => Err(error),
237 AttemptFateOutcome::Refused(refusal) => Err(SdkError::Protocol {
238 description: format!(
239 "subscription open failed ({error}) and the client authority \
240 refused the Failed fate: {refusal:?}"
241 ),
242 }),
243 },
244 }
245 }
246
247 pub fn recv_timeout(&self, timeout: Duration) -> Result<WebSocketDeliveredMessage, SdkError> {
258 self.mailbox.take(timeout)
259 }
260
261 pub fn stats(&self) -> Result<MailboxStats, SdkError> {
270 self.mailbox.stats()
271 }
272
273 #[must_use]
275 pub const fn subscription_id(&self) -> u64 {
276 self.subscription_id
277 }
278
279 #[must_use]
281 pub fn reconnect_state(&self) -> ReconnectState {
282 self.binding.lock().reconnect_state()
283 }
284
285 fn open_link(
288 address: &str,
289 channel: &str,
290 accepted_schemas: Vec<SchemaId>,
291 message_bound: usize,
292 auth_token: &[u8],
293 ) -> Result<
294 (
295 WsSocket,
296 WebSocketFrameDriver,
297 u64,
298 Vec<WebSocketDeliveredMessage>,
299 ),
300 SdkError,
301 > {
302 let mut driver = WebSocketFrameDriver::new();
303 let command = driver
304 .command_open()
305 .map_err(|refusal| SdkError::Protocol {
306 description: format!("subscription driver refused its first open: {refusal:?}"),
307 })?;
308 if command != SocketCommand::Open {
309 return Err(SdkError::Protocol {
310 description: "subscription driver emitted a non-open first command".to_string(),
311 });
312 }
313 let mut socket = WsSocket::connect(address, message_bound)?;
314 socket.set_read_timeout(Some(SETUP_TIMEOUT))?;
319 let step = driver.handle_event(SocketEvent::Opened);
320 if step.output != DriverOutput::Opened {
321 return Err(SdkError::Protocol {
322 description: format!("subscription driver refused the opened socket: {step:?}"),
323 });
324 }
325
326 let mut pending = Vec::new();
327 let connect = Frame::Connect {
328 flags: 0,
329 min_version: CLIENT_MIN_VERSION,
330 max_version: CLIENT_MAX_VERSION,
331 auth_token: auth_token.to_vec(),
332 };
333 match setup_exchange(&mut socket, &mut driver, &connect, &mut pending)? {
334 Frame::ConnectAck { .. } => {}
335 Frame::ConnectError {
336 reason_code,
337 message,
338 ..
339 } => {
340 return Err(connection_error(&format!(
341 "server rejected subscription connection (reason {reason_code}): {}",
342 message.unwrap_or_else(|| "no detail".to_string())
343 )));
344 }
345 other => {
346 return Err(unexpected_setup_frame("ConnectAck", &other));
347 }
348 }
349
350 let subscribe = Frame::Subscribe {
351 flags: 0,
352 stream_id: SUBSCRIPTION_STREAM_ID,
353 channel: channel.to_string(),
354 accepted_schemas,
355 max_in_flight: SUBSCRIBE_MAX_IN_FLIGHT,
356 };
357 let subscription_id =
358 match setup_exchange(&mut socket, &mut driver, &subscribe, &mut pending)? {
359 Frame::SubscribeAck {
360 subscription_id, ..
361 } => subscription_id,
362 Frame::SubscribeError {
363 reason_code,
364 message,
365 ..
366 } => {
367 return Err(SdkError::Protocol {
368 description: format!(
369 "server rejected subscribe (reason {reason_code}): {}",
370 message.unwrap_or_else(|| "no detail".to_string())
371 ),
372 });
373 }
374 other => {
375 return Err(unexpected_setup_frame("SubscribeAck", &other));
376 }
377 };
378 Ok((socket, driver, subscription_id, pending))
379 }
380
381 fn start(
383 socket: WsSocket,
384 driver: WebSocketFrameDriver,
385 binding: WebSocketAuthorityBinding,
386 subscription_id: u64,
387 pending: Vec<WebSocketDeliveredMessage>,
388 limits: MailboxLimits,
389 ) -> Result<Self, SdkError> {
390 socket.set_read_timeout(None)?;
396 let shutdown = socket.try_clone_stream()?;
397 let binding = Arc::new(Mutex::new(binding));
398 let reader_binding = Arc::clone(&binding);
399 let mailbox = Arc::new(Mailbox::new(limits, WEBSOCKET_SUBSCRIPTION_WORDS));
400 let reader_mailbox = Arc::clone(&mailbox);
401 let reader = std::thread::Builder::new()
402 .name("liminal-ws-subscription-reader".to_string())
403 .spawn(move || {
404 let end = run_reader(socket, driver, &reader_binding, &reader_mailbox, pending);
405 reader_mailbox.close(end);
406 })
407 .map_err(|source| SdkError::Protocol {
408 description: format!(
409 "failed to start websocket subscription reader thread: {source}"
410 ),
411 })?;
412 Ok(Self {
413 shutdown,
414 subscription_id,
415 mailbox,
416 binding,
417 reader: Some(reader),
418 })
419 }
420}
421
422impl Drop for WebSocketSubscriptionStream {
423 fn drop(&mut self) {
424 self.shutdown.shutdown(std::net::Shutdown::Both).ok();
429 self.mailbox.release();
432 if let Some(reader) = self.reader.take() {
433 reader.join().ok();
434 }
435 }
436}
437
438fn setup_exchange(
441 socket: &mut WsSocket,
442 driver: &mut WebSocketFrameDriver,
443 request: &Frame,
444 pending: &mut Vec<WebSocketDeliveredMessage>,
445) -> Result<Frame, SdkError> {
446 let bytes = super::encode_frame(request)?;
447 let command = driver
448 .command_send(bytes, ResponseExpectation::Correlated)
449 .map_err(|refusal| SdkError::Protocol {
450 description: format!("subscription driver refused the setup send: {refusal:?}"),
451 })?;
452 let SocketCommand::SendBinary(payload) = command else {
453 return Err(SdkError::Protocol {
454 description: "subscription driver emitted a non-send command for a send".to_string(),
455 });
456 };
457 if let Err(failure) = socket.send_binary(payload) {
458 let step = driver.handle_event(SocketEvent::Failed(failure));
459 if step.command == Some(SocketCommand::Close) {
460 socket.execute_close();
461 }
462 return Err(connection_error(&format!(
463 "failed to send subscription setup frame: {}",
464 socket
465 .last_failure_detail()
466 .unwrap_or("websocket send failed")
467 )));
468 }
469 loop {
470 let event = match socket.read_event() {
471 SocketRead::TimedOut => {
472 return Err(connection_error(
473 "subscription connection timed out waiting for a control-frame reply",
474 ));
475 }
476 SocketRead::Event(event) => event,
477 };
478 let step = driver.handle_event(event);
479 if step.command == Some(SocketCommand::Close) {
480 socket.execute_close();
481 }
482 match step.output {
483 DriverOutput::Frame { bytes, correlation } => {
484 let frame = decode_message(&bytes)?;
485 match correlation {
486 FrameCorrelation::UnsolicitedDelivery => {
487 if let Some(message) = delivered_message(frame) {
488 pending.push(message);
489 }
490 }
491 FrameCorrelation::CorrelatedResponse | FrameCorrelation::UnsolicitedFrame => {
492 return Ok(frame);
493 }
494 }
495 }
496 DriverOutput::Terminal(terminal) => {
497 return Err(connection_error(&format!(
498 "subscription connection terminated during setup: {terminal:?}"
499 )));
500 }
501 DriverOutput::Opened
502 | DriverOutput::PostTerminalIgnored(_)
503 | DriverOutput::Refused(_) => {
504 return Err(SdkError::Protocol {
505 description: format!(
506 "subscription driver produced an unexpected setup output: {:?}",
507 step.output
508 ),
509 });
510 }
511 }
512 }
513}
514
515fn run_reader(
523 mut socket: WsSocket,
524 mut driver: WebSocketFrameDriver,
525 binding: &Mutex<WebSocketAuthorityBinding>,
526 mailbox: &Mailbox<WebSocketDeliveredMessage>,
527 pending: Vec<WebSocketDeliveredMessage>,
528) -> ReaderEnd {
529 let mut first_error: Option<ReaderEnd> = None;
533 for message in pending {
534 if let Some(end) = offer_or_end(mailbox, message) {
535 close_link(&mut socket, &mut driver);
536 first_error = Some(end);
537 break;
538 }
539 }
540 loop {
541 let event = match socket.read_event() {
542 SocketRead::TimedOut => continue,
546 SocketRead::Event(event) => event,
547 };
548 let step = driver.handle_event(event);
549 if step.command == Some(SocketCommand::Close) {
550 socket.execute_close();
551 }
552 match step.output {
553 DriverOutput::Frame { bytes, correlation } => {
554 if first_error.is_some() {
555 continue;
558 }
559 first_error = handle_frame(&bytes, correlation, mailbox);
560 if first_error.is_some() {
561 close_link(&mut socket, &mut driver);
562 }
563 }
564 DriverOutput::Terminal(terminal) => {
565 let _outcome = binding.lock().established_terminal(&terminal);
567 return first_error.unwrap_or(ReaderEnd::Stopped);
568 }
569 DriverOutput::PostTerminalIgnored(_) => {
570 return first_error.unwrap_or(ReaderEnd::Stopped);
571 }
572 DriverOutput::Opened | DriverOutput::Refused(_) => {}
573 }
574 }
575}
576
577fn handle_frame(
581 bytes: &[u8],
582 correlation: FrameCorrelation,
583 mailbox: &Mailbox<WebSocketDeliveredMessage>,
584) -> Option<ReaderEnd> {
585 let frame = match decode_message(bytes) {
586 Ok(frame) => frame,
587 Err(error) => return Some(ReaderEnd::Failed(error)),
589 };
590 match (correlation, frame) {
591 (
592 FrameCorrelation::UnsolicitedDelivery,
593 Frame::Deliver {
594 delivery_seq,
595 envelope,
596 message,
597 ..
598 },
599 ) => offer_or_end(
600 mailbox,
601 WebSocketDeliveredMessage {
602 delivery_seq,
603 schema_id: envelope.schema_id,
604 payload: envelope.payload,
605 message,
606 },
607 ),
608 (_, Frame::Disconnect { .. }) => Some(ReaderEnd::Stopped),
611 (
620 _,
621 Frame::SubscribeError {
622 reason_code,
623 message,
624 ..
625 },
626 ) => Some(ReaderEnd::Failed(subscription_ended(reason_code, message))),
627 _ => mailbox.record_ignored_frame().err().map(ReaderEnd::Failed),
632 }
633}
634
635fn close_link(socket: &mut WsSocket, driver: &mut WebSocketFrameDriver) {
637 if driver.command_close().is_ok() {
638 socket.execute_close();
639 }
640}
641
642fn decode_message(bytes: &[u8]) -> Result<Frame, SdkError> {
644 match decode(bytes) {
645 Ok((frame, consumed)) if consumed == bytes.len() => Ok(frame),
646 Ok((_, consumed)) => Err(SdkError::Protocol {
647 description: format!(
648 "subscription decode consumed {consumed} of {} message bytes",
649 bytes.len()
650 ),
651 }),
652 Err(error) => Err(SdkError::Protocol {
653 description: format!("subscription wire codec error: {error}"),
654 }),
655 }
656}
657
658fn delivered_message(frame: Frame) -> Option<WebSocketDeliveredMessage> {
660 match frame {
661 Frame::Deliver {
662 delivery_seq,
663 envelope,
664 message,
665 ..
666 } => Some(WebSocketDeliveredMessage {
667 delivery_seq,
668 schema_id: envelope.schema_id,
669 payload: envelope.payload,
670 message,
671 }),
672 _ => None,
673 }
674}
675
676fn subscription_ended(reason_code: u16, message: Option<alloc::string::String>) -> SdkError {
682 SdkError::Protocol {
683 description: format!(
684 "server ended the subscription (reason {reason_code}): {}",
685 message.unwrap_or_else(|| "no detail".to_string())
686 ),
687 }
688}
689
690fn unexpected_setup_frame(expected: &str, actual: &Frame) -> SdkError {
692 SdkError::Protocol {
693 description: format!(
694 "expected {expected} during subscription setup, received {:?}",
695 actual.frame_type()
696 ),
697 }
698}
699
700#[cfg(test)]
701#[path = "subscription_decode_tests.rs"]
702mod decode_tests;
703
704#[cfg(test)]
705mod tests {
706 use super::SETUP_TIMEOUT;
707 use core::time::Duration;
708
709 #[test]
717 fn websocket_subscription_source_has_no_retired_reader_poll_family() {
718 const SOURCE: &str = include_str!("subscription.rs");
719 let production = SOURCE.split("#[cfg(test)]").next().unwrap_or(SOURCE);
720 for forbidden in [
721 "READER_POLL_TIMEOUT",
722 "AtomicBool",
723 "stop.load",
724 "stop.store",
725 "re-check the stop flag",
726 "poll the stop flag",
727 ] {
728 assert!(
729 !production.contains(forbidden),
730 "retired websocket-subscription-reader poll-family source \
731 `{forbidden}` reappeared"
732 );
733 }
734 }
735
736 #[test]
740 fn the_named_setup_deadline_is_the_ratified_five_seconds() {
741 assert_eq!(SETUP_TIMEOUT, Duration::from_secs(5));
742 }
743
744 const DELIVERED: liminal::protocol::DeliveredId = liminal::protocol::DeliveredId {
746 boot: 1_791_328_512_345,
747 id: 42,
748 };
749
750 fn deliver(message: Option<liminal::protocol::DeliveredId>) -> liminal::protocol::Frame {
751 liminal::protocol::Frame::Deliver {
752 flags: 0,
753 stream_id: super::SUBSCRIPTION_STREAM_ID,
754 delivery_seq: 1,
755 envelope: liminal::protocol::MessageEnvelope::new(
756 liminal::protocol::SchemaId::new([7; liminal::protocol::SchemaId::WIRE_LEN]),
757 liminal::protocol::CausalContext::independent(),
758 alloc::vec![1],
759 ),
760 message,
761 }
762 }
763
764 #[test]
766 fn the_subscription_offers_the_delivered_id_version() {
767 assert_eq!(
768 super::CLIENT_MAX_VERSION,
769 liminal::protocol::DELIVER_MESSAGE_ID_VERSION
770 );
771 }
772
773 #[test]
776 fn a_delivered_message_keeps_its_id_and_names_its_reply_parent() {
777 let with_id = super::delivered_message(deliver(Some(DELIVERED)));
778 assert_eq!(
779 with_id
780 .as_ref()
781 .and_then(super::WebSocketDeliveredMessage::message),
782 Some(DELIVERED)
783 );
784 assert_eq!(
785 with_id.and_then(|message| message.reply_parent()),
786 Some(liminal::protocol::MessageId::from("1791328512345:42"))
787 );
788 let without = super::delivered_message(deliver(None));
789 assert_eq!(
790 without
791 .as_ref()
792 .and_then(super::WebSocketDeliveredMessage::message),
793 None
794 );
795 assert_eq!(without.and_then(|message| message.reply_parent()), None);
796 }
797}