1use std::sync::{Arc, Mutex};
20use std::time::Duration;
21
22use crate::{
23 BlobDownload, BlobUploadGrant, SegmentFetch, SegmentRequest, Transport, TransportError,
24};
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
28pub enum RedirectPolicy {
29 #[default]
34 Deny,
35 Follow,
45}
46
47#[derive(Debug, Clone, Default)]
50pub struct HostTransportPolicy {
51 pub request_timeout: Option<Duration>,
53 pub round_deadline: Option<Duration>,
58 pub max_request_bytes: Option<u64>,
61 pub max_response_bytes: Option<u64>,
64 pub redirects: RedirectPolicy,
65}
66
67impl HostTransportPolicy {
68 pub fn validate(&self) -> Result<(), String> {
72 let now = std::time::Instant::now();
73 if let Some(timeout) = self.request_timeout {
74 if timeout.is_zero() || now.checked_add(timeout).is_none() {
75 return Err(
76 "sync.invalid_request: transport config requestTimeoutMs is out of range"
77 .to_owned(),
78 );
79 }
80 }
81 if let Some(deadline) = self.round_deadline {
82 if deadline.is_zero() || now.checked_add(deadline).is_none() {
83 return Err(
84 "sync.invalid_request: transport config roundDeadlineMs is out of range"
85 .to_owned(),
86 );
87 }
88 }
89 if self.max_request_bytes == Some(0) {
90 return Err(
91 "sync.invalid_request: transport config maxRequestBytes must be a positive integer"
92 .to_owned(),
93 );
94 }
95 if self.max_response_bytes == Some(0) {
96 return Err("sync.invalid_request: transport config maxResponseBytes must be a positive integer".to_owned());
97 }
98 Ok(())
99 }
100
101 pub fn from_config(config: &serde_json::Value) -> Result<Self, String> {
104 let bound = |key: &str, message: &'static str| -> Result<Option<u64>, String> {
105 match config.get(key) {
106 None | Some(serde_json::Value::Null) => Ok(None),
107 Some(value) => match value.as_u64() {
108 Some(number) if number >= 1 => Ok(Some(number)),
109 _ => Err(message.to_owned()),
110 },
111 }
112 };
113 let redirects = match config.get("redirects") {
114 None | Some(serde_json::Value::Null) => RedirectPolicy::Deny,
115 Some(serde_json::Value::String(value)) if value == "deny" => RedirectPolicy::Deny,
116 Some(serde_json::Value::String(value)) if value == "follow" => RedirectPolicy::Follow,
117 Some(_) => return Err(
118 "sync.invalid_request: transport config redirects must be \"deny\" or \"follow\""
119 .to_owned(),
120 ),
121 };
122 let policy = Self {
123 request_timeout: bound(
124 "requestTimeoutMs",
125 "sync.invalid_request: transport config requestTimeoutMs must be a positive integer",
126 )?
127 .map(Duration::from_millis),
128 round_deadline: bound(
129 "roundDeadlineMs",
130 "sync.invalid_request: transport config roundDeadlineMs must be a positive integer",
131 )?
132 .map(Duration::from_millis),
133 max_request_bytes: bound(
134 "maxRequestBytes",
135 "sync.invalid_request: transport config maxRequestBytes must be a positive integer",
136 )?,
137 max_response_bytes: bound(
138 "maxResponseBytes",
139 "sync.invalid_request: transport config maxResponseBytes must be a positive integer",
140 )?,
141 redirects,
142 };
143 policy.validate()?;
144 Ok(policy)
145 }
146}
147
148pub enum Inbound {
150 Text(String),
151 Binary(Vec<u8>),
152 Disconnected(TransportError),
155}
156
157pub struct InboundBuffer {
161 frames: Mutex<Vec<Inbound>>,
162 notify: Option<Arc<dyn Fn() + Send + Sync>>,
163}
164
165impl Default for InboundBuffer {
166 fn default() -> Self {
167 Self {
168 frames: Mutex::new(Vec::new()),
169 notify: None,
170 }
171 }
172}
173
174impl InboundBuffer {
175 pub fn with_notify(notify: Arc<dyn Fn() + Send + Sync>) -> Self {
176 Self {
177 frames: Mutex::new(Vec::new()),
178 notify: Some(notify),
179 }
180 }
181
182 pub fn push(&self, frame: Inbound) {
183 self.frames.lock().expect("inbound lock").push(frame);
184 if let Some(notify) = &self.notify {
185 notify();
186 }
187 }
188 fn take(&self) -> Vec<Inbound> {
189 std::mem::take(&mut *self.frames.lock().expect("inbound lock"))
190 }
191}
192
193#[allow(clippy::large_enum_variant)]
196pub enum HostTransport {
197 Null {
199 signed_urls: bool,
200 inbound: Arc<InboundBuffer>,
201 },
202 #[cfg(feature = "native-transport")]
203 Native(native::NativeTransport),
204}
205
206impl HostTransport {
207 pub fn from_config(config: &serde_json::Value) -> Result<Self, String> {
210 Self::new_from_config(config)
211 }
212
213 pub fn new_from_config(config: &serde_json::Value) -> Result<Self, String> {
216 Self::from_config_with_notify(config, None)
217 }
218
219 pub fn from_config_with_notify(
223 config: &serde_json::Value,
224 notify: Option<Arc<dyn Fn() + Send + Sync>>,
225 ) -> Result<Self, String> {
226 let policy = HostTransportPolicy::from_config(config)?;
227 Self::from_config_with_policy(config, policy, notify)
228 }
229
230 pub fn from_config_with_policy(
233 config: &serde_json::Value,
234 policy: HostTransportPolicy,
235 notify: Option<Arc<dyn Fn() + Send + Sync>>,
236 ) -> Result<Self, String> {
237 #[cfg(feature = "native-transport")]
238 {
239 if let Some(base_url) = config.get("baseUrl").and_then(|v| v.as_str()) {
240 return Ok(HostTransport::Native(native::NativeTransport::with_policy(
241 base_url, config, policy, notify,
242 )?));
243 }
244 }
245 #[cfg(not(feature = "native-transport"))]
246 {
247 policy.validate()?;
248 if config.get("baseUrl").is_some() {
249 return Err(
250 "this build has no native transport (rebuild with --features native-transport)"
251 .to_owned(),
252 );
253 }
254 }
255 Ok(HostTransport::Null {
256 signed_urls: false,
257 inbound: Arc::new(match notify {
258 Some(notify) => InboundBuffer::with_notify(notify),
259 None => InboundBuffer::default(),
260 }),
261 })
262 }
263
264 pub fn fork_round(&self) -> Self {
267 match self {
268 Self::Null {
269 signed_urls,
270 inbound,
271 } => Self::Null {
272 signed_urls: *signed_urls,
273 inbound: inbound.clone(),
274 },
275 #[cfg(feature = "native-transport")]
276 Self::Native(t) => Self::Native(t.fork_round()),
277 }
278 }
279
280 pub fn set_signed_urls(&mut self, value: bool) {
281 match self {
282 HostTransport::Null { signed_urls, .. } => *signed_urls = value,
283 #[cfg(feature = "native-transport")]
284 HostTransport::Native(t) => t.signed_urls = value,
285 }
286 }
287
288 pub fn set_headers(&mut self, headers: Vec<(String, String)>) {
291 match self {
292 HostTransport::Null { .. } => drop(headers),
293 #[cfg(feature = "native-transport")]
294 HostTransport::Native(t) => t.set_headers(headers),
295 }
296 }
297
298 pub fn set_policy(&mut self, policy: HostTransportPolicy) -> Result<(), String> {
303 policy.validate()?;
304 match self {
305 HostTransport::Null { .. } => {}
306 #[cfg(feature = "native-transport")]
307 HostTransport::Native(t) => t.set_policy(policy)?,
308 }
309 Ok(())
310 }
311
312 pub fn take_inbound(&mut self) -> Vec<Inbound> {
314 match self {
315 HostTransport::Null { inbound, .. } => inbound.take(),
316 #[cfg(feature = "native-transport")]
317 HostTransport::Native(t) => t.inbound.take(),
318 }
319 }
320
321 pub fn shutdown(&mut self) {
323 match self {
324 HostTransport::Null { .. } => {}
325 #[cfg(feature = "native-transport")]
326 HostTransport::Native(t) => t.shutdown(),
327 }
328 }
329}
330
331fn unavailable(op: &str) -> TransportError {
332 TransportError::new(
333 "transport.unavailable",
334 format!("{op} needs the native transport (build with --features native-transport)"),
335 )
336}
337
338#[cfg_attr(not(feature = "native-transport"), allow(unused_variables))]
341impl Transport for HostTransport {
342 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
343 match self {
344 HostTransport::Null { .. } => Err(unavailable("sync")),
345 #[cfg(feature = "native-transport")]
346 HostTransport::Native(t) => t.sync(request),
347 }
348 }
349
350 fn remote_operation(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
351 match self {
352 HostTransport::Null { .. } => Err(unavailable("remoteOperation")),
353 #[cfg(feature = "native-transport")]
354 HostTransport::Native(t) => t.remote_operation(request),
355 }
356 }
357
358 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
359 match self {
360 HostTransport::Null { .. } => Err(unavailable("realtimeSync")),
361 #[cfg(feature = "native-transport")]
362 HostTransport::Native(t) => t.realtime_sync(request),
363 }
364 }
365
366 fn download_segment(
367 &mut self,
368 request: &SegmentRequest,
369 on_progress: &mut dyn FnMut(u64),
370 ) -> Result<Vec<u8>, TransportError> {
371 match self {
372 HostTransport::Null { .. } => Err(unavailable("downloadSegment")),
373 #[cfg(feature = "native-transport")]
374 HostTransport::Native(t) => t.download_segment(request, on_progress),
375 }
376 }
377
378 fn supports_url_fetch(&self) -> bool {
379 match self {
380 HostTransport::Null { signed_urls, .. } => *signed_urls,
381 #[cfg(feature = "native-transport")]
382 HostTransport::Native(t) => t.signed_urls,
383 }
384 }
385
386 fn download_segments(
387 &mut self,
388 fetches: &[SegmentFetch],
389 on_progress: &mut (dyn FnMut(usize, u64) + Send),
390 ) -> Vec<Result<Vec<u8>, TransportError>> {
391 match self {
392 HostTransport::Null { .. } => fetches
393 .first()
394 .map(|_| Err(unavailable("downloadSegments")))
395 .into_iter()
396 .collect(),
397 #[cfg(feature = "native-transport")]
398 HostTransport::Native(t) => t.download_segments(fetches, on_progress),
399 }
400 }
401
402 fn fetch_url(
403 &mut self,
404 url: &str,
405 on_progress: &mut dyn FnMut(u64),
406 ) -> Result<Vec<u8>, TransportError> {
407 match self {
408 HostTransport::Null { .. } => Err(unavailable("fetchUrl")),
409 #[cfg(feature = "native-transport")]
410 HostTransport::Native(t) => t.fetch_url(url, on_progress),
411 }
412 }
413
414 fn blob_upload(
415 &mut self,
416 blob_id: &str,
417 bytes: &[u8],
418 media_type: Option<&str>,
419 ) -> Result<(), TransportError> {
420 match self {
421 HostTransport::Null { .. } => Err(unavailable("blobUpload")),
422 #[cfg(feature = "native-transport")]
423 HostTransport::Native(t) => t.blob_upload(blob_id, bytes, media_type),
424 }
425 }
426
427 fn blob_download(&mut self, blob_id: &str) -> Result<BlobDownload, TransportError> {
428 match self {
429 HostTransport::Null { .. } => Err(unavailable("blobDownload")),
430 #[cfg(feature = "native-transport")]
431 HostTransport::Native(t) => t.blob_download(blob_id),
432 }
433 }
434
435 fn fetch_blob_url(&mut self, url: &str) -> Result<Vec<u8>, TransportError> {
436 match self {
437 HostTransport::Null { .. } => Err(unavailable("fetchBlobUrl")),
438 #[cfg(feature = "native-transport")]
439 HostTransport::Native(t) => t.fetch_blob_url(url),
440 }
441 }
442
443 fn blob_upload_grant(
444 &mut self,
445 blob_id: &str,
446 byte_length: u64,
447 media_type: Option<&str>,
448 ) -> Result<BlobUploadGrant, TransportError> {
449 match self {
450 HostTransport::Null { .. } => Ok(BlobUploadGrant::None),
453 #[cfg(feature = "native-transport")]
454 HostTransport::Native(t) => t.blob_upload_grant(blob_id, byte_length, media_type),
455 }
456 }
457
458 fn blob_put_url(
459 &mut self,
460 url: &str,
461 bytes: &[u8],
462 media_type: Option<&str>,
463 ) -> Result<(), TransportError> {
464 match self {
465 HostTransport::Null { .. } => Err(unavailable("blobPutUrl")),
466 #[cfg(feature = "native-transport")]
467 HostTransport::Native(t) => t.blob_put_url(url, bytes, media_type),
468 }
469 }
470
471 fn realtime_connect(&mut self) -> Result<(), TransportError> {
472 match self {
473 HostTransport::Null { .. } => Err(unavailable("realtimeConnect")),
474 #[cfg(feature = "native-transport")]
475 HostTransport::Native(t) => {
476 t.set_realtime_client_id(None);
477 t.realtime_connect()
478 }
479 }
480 }
481
482 fn realtime_connect_for_client(&mut self, client_id: &str) -> Result<(), TransportError> {
483 match self {
484 HostTransport::Null { .. } => Err(unavailable("realtimeConnect")),
485 #[cfg(feature = "native-transport")]
486 HostTransport::Native(t) => {
487 t.set_realtime_client_id(Some(client_id));
488 t.realtime_connect()
489 }
490 }
491 }
492
493 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
494 match self {
495 HostTransport::Null { .. } => Err(unavailable("realtimeSend")),
496 #[cfg(feature = "native-transport")]
497 HostTransport::Native(t) => t.realtime_send(text),
498 }
499 }
500
501 fn realtime_close(&mut self) -> Result<(), TransportError> {
502 match self {
503 HostTransport::Null { .. } => Ok(()),
504 #[cfg(feature = "native-transport")]
505 HostTransport::Native(t) => t.realtime_close(),
506 }
507 }
508
509 fn round_deadline(&self) -> Option<std::time::Duration> {
510 match self {
511 HostTransport::Null { .. } => None,
512 #[cfg(feature = "native-transport")]
513 HostTransport::Native(t) => t.round_deadline(),
514 }
515 }
516
517 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
518 match self {
519 HostTransport::Null { .. } => {}
520 #[cfg(feature = "native-transport")]
521 HostTransport::Native(t) => t.set_round_deadline(deadline),
522 }
523 }
524}
525
526#[cfg(feature = "native-transport")]
527mod native {
528 use std::io::{self, Read, Write};
540 use std::net::{TcpStream, ToSocketAddrs};
541 use std::sync::atomic::{AtomicBool, Ordering};
542 use std::sync::{mpsc, Arc, Condvar, Mutex};
543 use std::thread::JoinHandle;
544 use std::time::{Duration, Instant};
545
546 use polling::{Event, Events, Poller};
547 use tungstenite::stream::MaybeTlsStream;
548 use tungstenite::Message;
549 use ureq::unversioned::resolver::DefaultResolver;
550 use ureq::unversioned::transport::{
551 time::Duration as UreqDuration, ConnectionDetails, Connector, DefaultConnector,
552 NextTimeout, Transport as UreqTransport,
553 };
554 use ureq::Timeout as UreqTimeout;
555
556 use super::{HostTransportPolicy, Inbound, InboundBuffer, RedirectPolicy};
557 use crate::{
558 BlobDownload, BlobUploadGrant, RealtimeRound, RoundInbound, SegmentFetch, SegmentRequest,
559 Transport, TransportError,
560 };
561
562 struct SocketRegistration {
564 poller: Arc<Poller>,
565 stream: TcpStream,
566 }
567
568 impl Drop for SocketRegistration {
569 fn drop(&mut self) {
570 let _ = self.poller.delete(&self.stream);
571 }
572 }
573
574 #[derive(Clone)]
575 struct SocketSender {
576 queue: mpsc::SyncSender<Outgoing>,
577 poller: Arc<Poller>,
578 }
579
580 struct Outgoing {
581 message: Message,
582 completed: mpsc::Sender<Result<(), TransportError>>,
583 }
584
585 const ROUND_IDLE_TIMEOUT: Duration = Duration::from_secs(30);
591 const REQUEST_CHUNK_BYTES: usize = 64 * 1024;
594
595 fn round_expiry(
598 last_progress: Instant,
599 idle: Duration,
600 ceiling: Option<Instant>,
601 ) -> (Instant, bool) {
602 let silent_at = last_progress + idle;
603 match ceiling {
604 Some(ceiling) if ceiling < silent_at => (ceiling, true),
605 _ => (silent_at, false),
606 }
607 }
608 #[derive(Default)]
615 pub(super) struct RoundChannel {
616 state: Mutex<RoundState>,
617 ready: Condvar,
618 }
619
620 #[derive(Default)]
621 struct RoundState {
622 round: RealtimeRound,
623 last_progress: Option<Instant>,
626 lost: bool,
628 outcome: Option<Result<Vec<u8>, TransportError>>,
630 }
631
632 impl RoundChannel {
633 fn begin(&self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
637 let mut state = self.state.lock().expect("round lock");
638 state.outcome = None;
639 let framed = state.round.begin(request)?;
640 state.last_progress = Some(Instant::now());
641 Ok(framed)
642 }
643
644 fn progressed(&self) {
646 self.state.lock().expect("round lock").last_progress = Some(Instant::now());
647 }
648
649 fn lost(&self) -> bool {
652 self.state.lock().expect("round lock").lost
653 }
654
655 fn route_binary(&self, frame: &[u8]) -> Option<Vec<u8>> {
659 let mut state = self.state.lock().expect("round lock");
660 match state.round.route_binary(frame) {
661 Ok(RoundInbound::Delta(body)) => Some(body),
662 Ok(RoundInbound::RoundProgress) => {
663 state.last_progress = Some(Instant::now());
664 None
665 }
666 Ok(RoundInbound::Ignored) => None,
667 Ok(RoundInbound::RoundComplete(bytes)) => {
668 state.outcome = Some(Ok(bytes));
669 self.ready.notify_all();
670 None
671 }
672 Err(error) => {
673 state.outcome = Some(Err(error));
674 self.ready.notify_all();
675 None
676 }
677 }
678 }
679
680 fn fail_in_flight(&self, error: TransportError) {
682 let mut state = self.state.lock().expect("round lock");
683 state.lost = true;
684 if state.round.in_flight() && state.outcome.is_none() {
685 state.round.abort();
686 state.outcome = Some(Err(error));
687 self.ready.notify_all();
688 }
689 }
690
691 fn wait(
694 &self,
695 ceiling: Option<Instant>,
696 idle: Duration,
697 ) -> Result<Vec<u8>, TransportError> {
698 let mut state = self.state.lock().expect("round lock");
699 while state.outcome.is_none() {
700 let (deadline, ceiling_binds) = round_expiry(
701 state.last_progress.unwrap_or_else(Instant::now),
702 idle,
703 ceiling,
704 );
705 let now = Instant::now();
706 if now >= deadline {
707 state.round.abort();
708 return Err(if ceiling_binds {
709 timeout_error("sync round deadline exceeded", "round_deadline")
710 } else {
711 TransportError::new(
712 "sync.transport_failed",
713 "realtime sync round made no progress (§8.7)",
714 )
715 });
716 }
717 let (guard, _timeout) = self
718 .ready
719 .wait_timeout(state, deadline - now)
720 .expect("round wait");
721 state = guard;
722 }
723 state.outcome.take().expect("outcome present")
724 }
725 }
726
727 fn transfer_error(
728 message: &'static str,
729 cause_kind: &'static str,
730 http_status: Option<u16>,
731 ) -> TransportError {
732 let mut details = serde_json::json!({ "causeKind": cause_kind });
733 if let Some(status) = http_status {
734 details["httpStatus"] = status.into();
735 }
736 let mut error = TransportError::new("transport.failed", message);
737 error.details = Some(details);
738 error
739 }
740
741 fn timeout_error(message: &'static str, cause_kind: &'static str) -> TransportError {
744 let mut error = TransportError::new("transport.timeout", message);
745 error.details = Some(serde_json::json!({ "causeKind": cause_kind }));
746 error
747 }
748
749 fn redirect_error(message: &'static str, http_status: Option<u16>) -> TransportError {
751 let mut details = serde_json::json!({ "causeKind": "redirect" });
752 if let Some(status) = http_status {
753 details["httpStatus"] = status.into();
754 }
755 let mut error = TransportError::new("transport.redirect", message);
756 error.details = Some(details);
757 error
758 }
759
760 fn redirect_refused(resp: &ureq::http::Response<ureq::Body>) -> Result<(), TransportError> {
765 let status = resp.status().as_u16();
766 if (300..400).contains(&status) {
767 return Err(redirect_error("redirect refused", Some(status)));
768 }
769 Ok(())
770 }
771
772 fn is_timeout_io(error: &std::io::Error) -> bool {
778 if error.kind() == std::io::ErrorKind::TimedOut {
779 return true;
780 }
781 let Some(inner) = error.get_ref() else {
782 return false;
783 };
784 let Some(ureq_error) = inner.downcast_ref::<ureq::Error>() else {
785 return false;
786 };
787 match ureq_error {
788 ureq::Error::Timeout(_) => true,
789 ureq::Error::Io(nested) => is_timeout_io(nested),
790 ureq::Error::Decompress(_, nested) => is_timeout_io(nested),
791 _ => false,
792 }
793 }
794
795 fn http_err(message: &'static str, error: ureq::Error) -> TransportError {
796 let (kind, status) = match error {
797 ureq::Error::StatusCode(status) if (300..400).contains(&status) => {
798 return redirect_error(message, Some(status))
799 }
800 ureq::Error::StatusCode(status) => ("status", Some(status)),
801 ureq::Error::Timeout(_) => return timeout_error(message, "timeout"),
802 ureq::Error::Io(error) => return io_err(message, error),
803 ureq::Error::HostNotFound
804 | ureq::Error::ConnectionFailed
805 | ureq::Error::ConnectProxyFailed(_) => ("connect", None),
806 ureq::Error::Tls(_)
807 | ureq::Error::Rustls(_)
808 | ureq::Error::Pem(_)
809 | ureq::Error::TlsRequired => ("tls", None),
810 ureq::Error::BadUri(_)
811 | ureq::Error::Http(_)
812 | ureq::Error::RequireHttpsOnly(_)
813 | ureq::Error::InvalidProxyUrl => ("request", None),
814 ureq::Error::Protocol(_) => ("protocol", None),
815 ureq::Error::BodyExceedsLimit(_)
816 | ureq::Error::BodyStalled
817 | ureq::Error::Decompress(_, _) => ("body", None),
818 ureq::Error::RedirectFailed | ureq::Error::TooManyRedirects => {
819 return redirect_error(message, None)
820 }
821 _ => ("unknown", None),
822 };
823 transfer_error(message, kind, status)
824 }
825
826 fn io_err(message: &'static str, error: std::io::Error) -> TransportError {
827 if is_timeout_io(&error) {
828 return timeout_error(message, "timeout");
829 }
830 let kind = match error.kind() {
831 std::io::ErrorKind::ConnectionRefused
832 | std::io::ErrorKind::ConnectionAborted
833 | std::io::ErrorKind::ConnectionReset
834 | std::io::ErrorKind::NotConnected => "connect",
835 _ => "io",
836 };
837 transfer_error(message, kind, None)
838 }
839
840 fn ws_err(message: &'static str, error: tungstenite::Error) -> TransportError {
841 match error {
842 tungstenite::Error::Io(error) => io_err(message, error),
843 tungstenite::Error::Tls(_) => transfer_error(message, "tls", None),
844 tungstenite::Error::Http(response) if response.status().is_redirection() => {
847 redirect_error(message, Some(response.status().as_u16()))
848 }
849 tungstenite::Error::Http(response) => {
850 transfer_error(message, "status", Some(response.status().as_u16()))
851 }
852 tungstenite::Error::Url(_) | tungstenite::Error::HttpFormat(_) => {
853 transfer_error(message, "request", None)
854 }
855 _ => transfer_error(message, "protocol", None),
856 }
857 }
858
859 const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
860
861 fn remaining(deadline: Instant) -> io::Result<Duration> {
862 deadline
863 .checked_duration_since(Instant::now())
864 .filter(|left| !left.is_zero())
865 .ok_or_else(|| {
866 io::Error::new(
867 io::ErrorKind::TimedOut,
868 "websocket establishment deadline exceeded",
869 )
870 })
871 }
872
873 struct EstablishingStream {
874 socket: TcpStream,
875 deadline: Option<Instant>,
876 }
877
878 impl EstablishingStream {
879 fn timeout(&self) -> io::Result<Option<Duration>> {
880 self.deadline.map(remaining).transpose()
881 }
882 fn established(&mut self) -> io::Result<TcpStream> {
883 self.deadline = None;
884 self.socket.set_read_timeout(None)?;
885 self.socket.set_write_timeout(None)?;
886 self.socket.try_clone()
887 }
888 }
889
890 fn establishment_io<T>(result: io::Result<T>, establishing: bool) -> io::Result<T> {
891 result.map_err(|error| {
892 if establishing
893 && matches!(
894 error.kind(),
895 io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut
896 )
897 {
898 io::Error::new(
899 io::ErrorKind::TimedOut,
900 "websocket establishment deadline exceeded",
901 )
902 } else {
903 error
904 }
905 })
906 }
907
908 impl Read for EstablishingStream {
909 fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
910 if self.deadline.is_some() {
911 self.socket.set_read_timeout(self.timeout()?)?;
912 }
913 establishment_io(self.socket.read(bytes), self.deadline.is_some())
914 }
915 }
916 impl Write for EstablishingStream {
917 fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
918 if self.deadline.is_some() {
919 self.socket.set_write_timeout(self.timeout()?)?;
920 }
921 establishment_io(self.socket.write(bytes), self.deadline.is_some())
922 }
923 fn flush(&mut self) -> io::Result<()> {
924 if self.deadline.is_some() {
925 self.socket.set_write_timeout(self.timeout()?)?;
926 }
927 establishment_io(self.socket.flush(), self.deadline.is_some())
928 }
929 }
930
931 fn connect_websocket(
932 request: tungstenite::handshake::client::Request,
933 deadline: Instant,
934 ) -> Result<tungstenite::WebSocket<MaybeTlsStream<EstablishingStream>>, TransportError> {
935 let host = request
936 .uri()
937 .host()
938 .ok_or_else(|| transfer_error("realtime URL invalid", "request", None))?;
939 let host = host.trim_start_matches('[').trim_end_matches(']');
940 let port = match request.uri().scheme_str() {
941 Some("ws") => request.uri().port_u16().unwrap_or(80),
942 Some("wss") => request.uri().port_u16().unwrap_or(443),
943 _ => return Err(transfer_error("realtime URL invalid", "request", None)),
944 };
945 let host = host.to_owned();
948 let (resolved, addresses) = mpsc::sync_channel(1);
949 std::thread::Builder::new()
950 .name("syncular-dns".to_owned())
951 .spawn(move || {
952 let result = (host.as_str(), port)
953 .to_socket_addrs()
954 .map(|items| items.collect::<Vec<_>>());
955 let _ = resolved.send(result);
956 })
957 .map_err(|e| io_err("realtime DNS failed", e))?;
958 let addresses = addresses
959 .recv_timeout(remaining(deadline).map_err(|e| io_err("realtime DNS failed", e))?)
960 .map_err(|e| match e {
961 mpsc::RecvTimeoutError::Timeout => {
962 timeout_error("realtime DNS timed out", "timeout")
963 }
964 mpsc::RecvTimeoutError::Disconnected => {
965 transfer_error("realtime DNS failed", "io", None)
966 }
967 })?
968 .map_err(|e| io_err("realtime DNS failed", e))?;
969 let mut last_error = io::Error::new(io::ErrorKind::NotFound, "DNS returned no addresses");
970 let mut socket = None;
971 for address in addresses {
972 match TcpStream::connect_timeout(
973 &address,
974 remaining(deadline).map_err(|e| io_err("realtime TCP connect failed", e))?,
975 ) {
976 Ok(stream) => {
977 socket = Some(stream);
978 break;
979 }
980 Err(error) => last_error = error,
981 }
982 }
983 let socket = socket.ok_or_else(|| io_err("realtime TCP connect failed", last_error))?;
984 socket
985 .set_nodelay(true)
986 .map_err(|e| io_err("realtime TCP connect failed", e))?;
987 let stream = EstablishingStream {
988 socket,
989 deadline: Some(deadline),
990 };
991 let (ws, _) = tungstenite::client_tls_with_config(request, stream, None, None).map_err(
994 |e| match e {
995 tungstenite::HandshakeError::Failure(error) => {
996 ws_err("realtime connection failed", error)
997 }
998 tungstenite::HandshakeError::Interrupted(_) => {
1001 transfer_error("realtime connection failed", "protocol", None)
1002 }
1003 },
1004 )?;
1005 remaining(deadline).map_err(|e| io_err("realtime connection failed", e))?;
1006 Ok(ws)
1007 }
1008
1009 #[cfg(test)]
1010 mod establishment_tests {
1011 use super::*;
1012 use std::net::{Shutdown, TcpListener};
1013 use tungstenite::client::IntoClientRequest;
1014
1015 fn withheld_response(tls: bool, old: bool) {
1018 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1019 let url = format!(
1020 "{}://{}/realtime",
1021 if tls { "wss" } else { "ws" },
1022 listener.local_addr().unwrap()
1023 );
1024 let (accepted_tx, accepted_rx) = mpsc::channel();
1025 let (release_tx, release_rx) = mpsc::channel();
1026 let peer = std::thread::spawn(move || {
1027 let (mut stream, _) = listener.accept().unwrap();
1028 stream
1029 .set_read_timeout(Some(Duration::from_secs(2)))
1030 .unwrap();
1031 let mut bytes = [0; 4096];
1032 assert!(stream.read(&mut bytes).unwrap() > 0);
1033 accepted_tx.send(()).unwrap();
1034 release_rx.recv_timeout(Duration::from_secs(3)).unwrap();
1035 let _ = stream.shutdown(Shutdown::Both);
1036 });
1037 let (done_tx, done_rx) = mpsc::channel();
1038 let worker = std::thread::spawn(move || {
1039 let result = if old {
1040 tungstenite::connect(url)
1041 .map(|_| ())
1042 .map_err(|e| e.to_string())
1043 } else {
1044 connect_websocket(
1045 url.into_client_request().unwrap(),
1046 Instant::now() + Duration::from_millis(250),
1047 )
1048 .map(|_| ())
1049 .map_err(|e| format!("{}: {}", e.code, e.message))
1050 };
1051 done_tx.send(result).unwrap();
1052 });
1053 accepted_rx.recv_timeout(Duration::from_secs(2)).unwrap();
1054 if old {
1055 assert!(
1056 matches!(
1057 done_rx.recv_timeout(Duration::from_millis(500)),
1058 Err(mpsc::RecvTimeoutError::Timeout)
1059 ),
1060 "original connect unexpectedly had a deadline"
1061 );
1062 release_tx.send(()).unwrap();
1063 assert!(done_rx
1064 .recv_timeout(Duration::from_secs(2))
1065 .unwrap()
1066 .is_err());
1067 } else {
1068 let error = done_rx
1069 .recv_timeout(Duration::from_secs(2))
1070 .expect("establishment remained blocked")
1071 .unwrap_err();
1072 assert_eq!(error, "transport.timeout: realtime connection failed");
1073 eprintln!(
1074 "withheld {} produces {error}",
1075 if tls { "TLS" } else { "upgrade" }
1076 );
1077 release_tx.send(()).unwrap();
1078 }
1079 worker.join().unwrap();
1080 peer.join().unwrap();
1081 }
1082
1083 #[test]
1084 fn original_connect_blocks_on_withheld_upgrade_and_tls() {
1085 withheld_response(false, true);
1086 withheld_response(true, true);
1087 }
1088 #[test]
1089 fn establishment_deadline_errors_on_withheld_upgrade_and_tls() {
1090 withheld_response(false, false);
1091 withheld_response(true, false);
1092 }
1093
1094 #[test]
1095 fn slow_upgrade_bytes_cannot_renew_the_absolute_deadline() {
1096 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1097 let url = format!("ws://{}", listener.local_addr().unwrap());
1098 let peer = std::thread::spawn(move || {
1099 let (mut stream, _) = listener.accept().unwrap();
1100 let mut bytes = [0; 4096];
1101 let _ = stream.read(&mut bytes).unwrap();
1102 for byte in b"HTTP/1.1 101 Switching Protocols\r\n" {
1103 if stream.write_all(&[*byte]).is_err() {
1104 break;
1105 }
1106 std::thread::sleep(Duration::from_millis(40));
1107 }
1108 });
1109 let started = Instant::now();
1110 assert!(connect_websocket(
1111 url.into_client_request().unwrap(),
1112 started + Duration::from_millis(250)
1113 )
1114 .is_err());
1115 assert!(started.elapsed() < Duration::from_secs(1));
1116 peer.join().unwrap();
1117 }
1118
1119 #[test]
1120 fn saturated_tcp_accept_queue_returns_an_observable_error() {
1121 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1124 let address = listener.local_addr().unwrap();
1125 let mut queued = Vec::new();
1126 loop {
1127 match TcpStream::connect_timeout(&address, Duration::from_millis(50)) {
1128 Ok(stream) => queued.push(stream),
1129 Err(error) => {
1130 assert!(
1131 matches!(
1132 error.kind(),
1133 io::ErrorKind::TimedOut
1134 | io::ErrorKind::WouldBlock
1135 | io::ErrorKind::ConnectionRefused
1136 | io::ErrorKind::ConnectionReset
1137 ),
1138 "{error:?}"
1139 );
1140 break;
1141 }
1142 }
1143 assert!(queued.len() < 4096, "listener queue did not saturate");
1144 }
1145 let started = Instant::now();
1146 let url = format!("ws://{address}");
1147 let error = match connect_websocket(
1148 url.into_client_request().unwrap(),
1149 started + Duration::from_millis(250),
1150 ) {
1151 Err(error) => error,
1152 Ok(_) => panic!("full accept queue unexpectedly connected"),
1153 };
1154 assert!(
1155 (error.code == "transport.timeout" || error.code == "transport.failed")
1156 && (error.message == "realtime TCP connect failed"
1157 || error.message == "realtime connection failed"),
1158 "{}: {}",
1159 error.code,
1160 error.message
1161 );
1162 assert!(started.elapsed() < Duration::from_secs(2));
1163 }
1164 }
1165
1166 fn segment_error(mut error: TransportError) -> TransportError {
1170 if error.code == "transport.failed" {
1171 error.code = "sync.transport_failed".into();
1172 error.message = "segment transfer failed".into();
1173 }
1174 error
1175 }
1176
1177 fn is_would_block(e: &tungstenite::Error) -> bool {
1179 matches!(
1180 e,
1181 tungstenite::Error::Io(io) if matches!(
1182 io.kind(),
1183 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
1184 )
1185 )
1186 }
1187
1188 pub struct NativeTransport {
1189 base_url: String,
1190 ws_url: String,
1191 headers: Vec<(String, String)>,
1193 base_url_has_userinfo: bool,
1197 agent: ureq::Agent,
1198 policy: HostTransportPolicy,
1199 round_deadline_at: Option<std::time::Instant>,
1203 pub signed_urls: bool,
1204 pub inbound: Arc<InboundBuffer>,
1205 outgoing: Option<Box<SocketSender>>,
1207 reader: Option<JoinHandle<()>>,
1208 reader_stop: Arc<AtomicBool>,
1209 round: Arc<RoundChannel>,
1211 realtime_client_id: Option<String>,
1212 }
1213
1214 #[derive(Debug)]
1221 struct IdleConnector(Option<Duration>);
1222
1223 impl Connector<Box<dyn UreqTransport>> for IdleConnector {
1224 type Out = Box<dyn UreqTransport>;
1225
1226 fn connect(
1227 &self,
1228 _: &ConnectionDetails,
1229 chained: Option<Box<dyn UreqTransport>>,
1230 ) -> Result<Option<Self::Out>, ureq::Error> {
1231 Ok(chained.map(|inner| match self.0 {
1232 Some(idle) => IdleTransport { inner, idle }.boxed(),
1233 None => inner,
1234 }))
1235 }
1236 }
1237
1238 #[derive(Debug)]
1239 struct IdleTransport {
1240 inner: Box<dyn UreqTransport>,
1241 idle: Duration,
1242 }
1243
1244 fn idle_bound(timeout: NextTimeout, idle: Duration) -> NextTimeout {
1247 let idle = UreqDuration::Exact(idle);
1248 if timeout.after.is_not_happening() || timeout.after.as_nanos() > idle.as_nanos() {
1249 NextTimeout {
1250 after: idle,
1251 reason: UreqTimeout::RecvBody,
1252 }
1253 } else {
1254 timeout
1255 }
1256 }
1257
1258 impl UreqTransport for IdleTransport {
1259 fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers {
1260 self.inner.buffers()
1261 }
1262 fn transmit_output(
1263 &mut self,
1264 amount: usize,
1265 timeout: NextTimeout,
1266 ) -> Result<(), ureq::Error> {
1267 self.inner
1268 .transmit_output(amount, idle_bound(timeout, self.idle))
1269 }
1270 fn await_input(&mut self, timeout: NextTimeout) -> Result<bool, ureq::Error> {
1271 self.inner.await_input(idle_bound(timeout, self.idle))
1272 }
1273 fn is_open(&mut self) -> bool {
1274 self.inner.is_open()
1275 }
1276 fn is_tls(&self) -> bool {
1277 self.inner.is_tls()
1278 }
1279 }
1280
1281 fn build_agent(policy: &HostTransportPolicy) -> ureq::Agent {
1282 let config = ureq::Agent::config_builder()
1283 .timeout_resolve(policy.request_timeout)
1284 .timeout_connect(policy.request_timeout)
1285 .timeout_send_request(policy.request_timeout)
1286 .timeout_recv_response(policy.request_timeout)
1287 .max_redirects(match policy.redirects {
1288 RedirectPolicy::Deny => 0,
1289 RedirectPolicy::Follow => 10,
1290 })
1291 .max_redirects_will_error(matches!(policy.redirects, RedirectPolicy::Follow))
1292 .redirect_auth_headers(ureq::config::RedirectAuthHeaders::Never)
1293 .http_status_as_error(false)
1298 .build();
1299 ureq::Agent::with_parts(
1300 config,
1301 DefaultConnector::new().chain(IdleConnector(policy.request_timeout)),
1302 DefaultResolver::default(),
1303 )
1304 }
1305
1306 fn derive_ws_url(base_url: &str) -> String {
1307 let ws = if let Some(rest) = base_url.strip_prefix("https://") {
1310 format!("wss://{rest}")
1311 } else if let Some(rest) = base_url.strip_prefix("http://") {
1312 format!("ws://{rest}")
1313 } else {
1314 base_url.to_owned()
1315 };
1316 let trimmed = ws.trim_end_matches('/');
1317 format!("{trimmed}/realtime")
1318 }
1319
1320 impl NativeTransport {
1321 pub fn fork_round(&self) -> Self {
1322 Self {
1323 base_url: self.base_url.clone(),
1324 ws_url: self.ws_url.clone(),
1325 headers: self.headers.clone(),
1326 base_url_has_userinfo: self.base_url_has_userinfo,
1327 agent: self.agent.clone(),
1328 policy: self.policy.clone(),
1329 round_deadline_at: None,
1330 signed_urls: self.signed_urls,
1331 inbound: self.inbound.clone(),
1332 outgoing: self.outgoing.clone(),
1333 reader: None,
1334 reader_stop: self.reader_stop.clone(),
1335 round: self.round.clone(),
1336 realtime_client_id: self.realtime_client_id.clone(),
1337 }
1338 }
1339
1340 pub fn new(
1341 base_url: &str,
1342 config: &serde_json::Value,
1343 notify: Option<Arc<dyn Fn() + Send + Sync>>,
1344 ) -> Result<Self, String> {
1345 let policy = HostTransportPolicy::from_config(config)?;
1346 Self::with_policy(base_url, config, policy, notify)
1347 }
1348
1349 pub fn with_policy(
1350 base_url: &str,
1351 config: &serde_json::Value,
1352 policy: HostTransportPolicy,
1353 notify: Option<Arc<dyn Fn() + Send + Sync>>,
1354 ) -> Result<Self, String> {
1355 policy.validate()?;
1356 let mut headers = Vec::new();
1357 if let Some(map) = config.get("headers").and_then(|v| v.as_object()) {
1358 for (k, v) in map {
1359 if let Some(s) = v.as_str() {
1360 headers.push((k.clone(), s.to_owned()));
1361 }
1362 }
1363 }
1364 let ws_url = config
1365 .get("wsUrl")
1366 .and_then(|v| v.as_str())
1367 .map(str::to_owned)
1368 .unwrap_or_else(|| derive_ws_url(base_url));
1369 let base_url_has_userinfo = url::Url::parse(base_url)
1372 .map(|url| !url.username().is_empty() || url.password().is_some())
1373 .unwrap_or(false);
1374 Ok(NativeTransport {
1375 base_url: base_url.trim_end_matches('/').to_owned(),
1376 ws_url,
1377 headers,
1378 base_url_has_userinfo,
1379 agent: build_agent(&policy),
1380 policy,
1381 round_deadline_at: None,
1382 signed_urls: false,
1383 inbound: Arc::new(match notify {
1384 Some(notify) => InboundBuffer::with_notify(notify),
1385 None => InboundBuffer::default(),
1386 }),
1387 outgoing: None,
1388 reader: None,
1389 reader_stop: Arc::new(AtomicBool::new(false)),
1390 round: Arc::new(RoundChannel::default()),
1391 realtime_client_id: None,
1392 })
1393 }
1394
1395 pub fn set_policy(&mut self, policy: HostTransportPolicy) -> Result<(), String> {
1399 policy.validate()?;
1400 self.agent = build_agent(&policy);
1401 self.policy = policy;
1402 Ok(())
1403 }
1404
1405 pub fn round_deadline(&self) -> Option<Duration> {
1406 self.policy.round_deadline
1407 }
1408
1409 pub fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
1410 self.round_deadline_at = deadline;
1411 }
1412
1413 fn round_remaining(&self) -> Result<Option<Duration>, TransportError> {
1418 match self.round_deadline_at {
1419 Some(deadline) => deadline
1420 .checked_duration_since(std::time::Instant::now())
1421 .map(Some)
1422 .ok_or_else(|| timeout_error("sync round deadline exceeded", "round_deadline")),
1423 None => Ok(None),
1424 }
1425 }
1426
1427 fn credentialed(&self) -> bool {
1430 !self.headers.is_empty() || self.base_url_has_userinfo
1431 }
1432
1433 fn finish_request<B>(
1439 &self,
1440 req: ureq::RequestBuilder<B>,
1441 credentialed: bool,
1442 ) -> Result<ureq::RequestBuilder<B>, TransportError> {
1443 let timeout = self.round_remaining()?;
1444 let follow = self.policy.redirects == RedirectPolicy::Follow && !credentialed;
1445 Ok(req
1446 .config()
1447 .timeout_global(timeout)
1448 .max_redirects(if follow { 10 } else { 0 })
1449 .max_redirects_will_error(true)
1450 .build())
1451 }
1452
1453 fn check_request_bytes(&self, len: u64) -> Result<(), TransportError> {
1455 match self.policy.max_request_bytes {
1456 Some(max) if len > max => {
1457 let mut error = TransportError::new(
1458 "transport.request_too_large",
1459 "request body exceeds the configured transport limit",
1460 );
1461 error.details = Some(serde_json::json!({ "limit": max, "size": len }));
1462 Err(error)
1463 }
1464 _ => Ok(()),
1465 }
1466 }
1467
1468 fn post_sync(&self, path: &str, body: &[u8]) -> Result<Vec<u8>, TransportError> {
1469 self.check_request_bytes(body.len() as u64)?;
1470 let credentialed = self.credentialed();
1471 let url = format!("{}{}", self.base_url, path);
1472 let mut req = self
1475 .agent
1476 .post(&url)
1477 .header("content-type", "application/vnd.syncular.sync.v2");
1478 for (k, v) in &self.headers {
1479 req = req.header(k.as_str(), v.as_str());
1480 }
1481 let req = self.finish_request(req, credentialed)?;
1482 let resp = req
1483 .send(body)
1484 .map_err(|e| http_err("sync request failed", e))?;
1485 read_body(resp, None, self.policy.max_response_bytes)
1486 }
1487
1488 fn post_operation(&self, body: &[u8]) -> Result<Vec<u8>, TransportError> {
1489 self.check_request_bytes(body.len() as u64)?;
1490 let credentialed = self.credentialed();
1491 let url = format!("{}/operations", self.base_url);
1492 let mut req = self.agent.post(&url).header(
1493 "content-type",
1494 "application/vnd.syncular.operations.v1+json",
1495 );
1496 for (key, value) in &self.headers {
1497 req = req.header(key.as_str(), value.as_str());
1498 }
1499 let req = self.finish_request(req, credentialed)?;
1500 let response = req
1501 .send(body)
1502 .map_err(|error| http_err("remote operation request failed", error))?;
1503 read_body(response, None, self.policy.max_response_bytes)
1504 }
1505
1506 fn get_bytes(&self, url: &str, with_headers: bool) -> Result<Vec<u8>, TransportError> {
1507 let mut req = self.agent.get(url);
1510 if with_headers {
1511 for (k, v) in &self.headers {
1512 req = req.header(k.as_str(), v.as_str());
1513 }
1514 }
1515 let req = self.finish_request(req, true)?;
1516 let resp = req.call().map_err(|e| http_err("URL fetch failed", e))?;
1517 read_body(resp, None, self.policy.max_response_bytes)
1518 }
1519
1520 pub fn set_headers(&mut self, headers: Vec<(String, String)>) {
1521 self.headers = headers;
1522 }
1523
1524 pub fn set_realtime_client_id(&mut self, client_id: Option<&str>) {
1525 self.realtime_client_id = client_id.map(str::to_owned);
1526 }
1527
1528 pub fn shutdown(&mut self) {
1529 self.reader_stop.store(true, Ordering::SeqCst);
1530 self.round.fail_in_flight(TransportError::new(
1533 "sync.transport_failed",
1534 "realtime disconnected mid-round (§8.7)",
1535 ));
1536 if let Some(outgoing) = &self.outgoing {
1537 let _ = outgoing.poller.notify();
1538 }
1539 if let Some(handle) = self.reader.take() {
1540 let _ = handle.join();
1541 }
1542 self.outgoing = None;
1543 }
1544
1545 fn send_message(&mut self, message: Message) -> Result<(), TransportError> {
1546 let (wait, round_bound) = match self.round_deadline_at {
1551 Some(deadline) => {
1552 match deadline.checked_duration_since(std::time::Instant::now()) {
1553 Some(remaining) if remaining < ROUND_IDLE_TIMEOUT => (remaining, true),
1554 Some(_) => (ROUND_IDLE_TIMEOUT, false),
1555 None => {
1558 return Err(timeout_error("realtime send timed out", "round_deadline"));
1559 }
1560 }
1561 }
1562 None => (ROUND_IDLE_TIMEOUT, false),
1563 };
1564 let result = (|| {
1565 let Some(outgoing) = &self.outgoing else {
1566 return Err(TransportError::new(
1567 "transport.failed",
1568 "realtime not connected",
1569 ));
1570 };
1571 let (completed, result) = mpsc::channel();
1572 outgoing
1573 .queue
1574 .try_send(Outgoing { message, completed })
1575 .map_err(|_| {
1576 TransportError::new("transport.failed", "realtime send queue unavailable")
1577 })?;
1578 outgoing
1579 .poller
1580 .notify()
1581 .map_err(|e| io_err("realtime wake failed", e))?;
1582 result.recv_timeout(wait).map_err(|_| {
1583 if round_bound {
1584 timeout_error("realtime send timed out", "round_deadline")
1585 } else {
1586 TransportError::new("transport.failed", "realtime send did not complete")
1587 }
1588 })?
1589 })();
1590 if let Err(error) = &result {
1593 self.shutdown();
1594 self.inbound.push(Inbound::Disconnected(error.clone()));
1597 }
1598 result
1599 }
1600 }
1601
1602 fn http_status_body_error(status: u16, body: &[u8]) -> TransportError {
1606 let mut error = TransportError::new("transport.failed", "HTTP request failed");
1607 let mut details = serde_json::Map::new();
1608 if let Ok(serde_json::Value::Object(parsed)) =
1609 serde_json::from_slice::<serde_json::Value>(body)
1610 {
1611 if let Some(code) = parsed.get("code").and_then(serde_json::Value::as_str) {
1612 if !code.is_empty() {
1613 error.code = code.to_owned();
1614 }
1615 }
1616 if let Some(message) = parsed.get("message").and_then(serde_json::Value::as_str) {
1617 error.message = message.to_owned();
1618 }
1619 for key in ["category", "recommendedAction"] {
1620 if let Some(value) = parsed.get(key).and_then(serde_json::Value::as_str) {
1621 details.insert(key.to_owned(), serde_json::Value::from(value));
1622 }
1623 }
1624 if let Some(retryable) = parsed.get("retryable").and_then(serde_json::Value::as_bool) {
1625 details.insert("retryable".to_owned(), serde_json::Value::from(retryable));
1626 }
1627 if let Some(serde_json::Value::Object(inner)) = parsed.get("details") {
1628 for (key, value) in inner {
1629 details.insert(key.clone(), value.clone());
1630 }
1631 }
1632 }
1633 details.insert("causeKind".to_owned(), serde_json::Value::from("status"));
1634 details.insert("httpStatus".to_owned(), serde_json::Value::from(status));
1635 error.details = Some(serde_json::Value::Object(details));
1636 error
1637 }
1638
1639 fn read_body(
1640 resp: ureq::http::Response<ureq::Body>,
1641 mut progress: Option<&mut dyn FnMut(u64)>,
1642 max: Option<u64>,
1643 ) -> Result<Vec<u8>, TransportError> {
1644 redirect_refused(&resp)?;
1645 use std::io::Read;
1646 let status = resp.status().as_u16();
1647 let mut reader = resp.into_body().into_with_config().reader();
1648 let mut bytes = Vec::new();
1649 let mut chunk = [0u8; 64 * 1024];
1650 let mut reported = 0;
1651 loop {
1652 let n = reader.read(&mut chunk).map_err(|error| {
1653 if is_timeout_io(&error) {
1654 timeout_error("response body read times out", "timeout")
1655 } else {
1656 transfer_error("response body read failed", "body", Some(status))
1657 }
1658 })?;
1659 if n == 0 {
1660 break;
1661 }
1662 if let Some(max) = max {
1665 if bytes.len() as u64 + n as u64 > max {
1666 return Err(response_too_large(max));
1667 }
1668 }
1669 bytes.extend_from_slice(&chunk[..n]);
1670 if bytes.len() - reported >= 64 * 1024 {
1671 if let Some(callback) = progress.as_mut() {
1672 callback(bytes.len() as u64);
1673 }
1674 reported = bytes.len();
1675 }
1676 }
1677 if bytes.len() != reported {
1678 if let Some(callback) = progress {
1679 callback(bytes.len() as u64);
1680 }
1681 }
1682 if status >= 400 {
1683 return Err(http_status_body_error(status, &bytes));
1684 }
1685 Ok(bytes)
1686 }
1687
1688 fn response_too_large(max: u64) -> TransportError {
1689 let mut error = TransportError::new(
1690 "transport.response_too_large",
1691 "response body exceeds the configured transport limit",
1692 );
1693 error.details = Some(serde_json::json!({ "limit": max }));
1694 error
1695 }
1696
1697 impl Transport for NativeTransport {
1698 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1699 self.post_sync("/sync", request)
1700 }
1701
1702 fn remote_operation(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1703 self.post_operation(request)
1704 }
1705
1706 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1707 if self.outgoing.is_none() {
1717 return self.post_sync("/sync", request);
1718 }
1719 self.round_remaining()?;
1720 let framed = self.round.begin(request)?;
1721 let (tag, body) = framed.split_first().expect("framed request carries a tag");
1722 for chunk in body.chunks(REQUEST_CHUNK_BYTES) {
1723 let mut message = Vec::with_capacity(chunk.len() + 1);
1724 message.push(*tag);
1725 message.extend_from_slice(chunk);
1726 if let Err(error) = self.send_message(Message::Binary(message.into())) {
1727 self.round.fail_in_flight(error);
1728 break;
1729 }
1730 self.round.progressed();
1731 }
1732 let outcome = self.round.wait(self.round_deadline_at, ROUND_IDLE_TIMEOUT);
1733 if let Err(error) = &outcome {
1738 if !self.round.lost() {
1739 self.shutdown();
1740 self.inbound.push(Inbound::Disconnected(error.clone()));
1741 }
1742 }
1743 outcome
1744 }
1745
1746 fn download_segment(
1747 &mut self,
1748 request: &SegmentRequest,
1749 on_progress: &mut dyn FnMut(u64),
1750 ) -> Result<Vec<u8>, TransportError> {
1751 let url = format!("{}/segments/{}", self.base_url, request.segment_id);
1758 let credentialed = self.credentialed();
1759 let mut req = self
1760 .agent
1761 .get(&url)
1762 .header("x-syncular-scopes", &request.requested_scopes_json);
1763 for (k, v) in &self.headers {
1764 req = req.header(k.as_str(), v.as_str());
1765 }
1766 let req = self.finish_request(req, credentialed)?;
1767 let resp = req
1768 .call()
1769 .map_err(|error| segment_error(http_err("segment request failed", error)))?;
1770 read_body(resp, Some(on_progress), self.policy.max_response_bytes)
1771 .map_err(segment_error)
1772 }
1773
1774 fn supports_url_fetch(&self) -> bool {
1775 self.signed_urls
1776 }
1777
1778 fn download_segments(
1779 &mut self,
1780 fetches: &[SegmentFetch],
1781 on_progress: &mut (dyn FnMut(usize, u64) + Send),
1782 ) -> Vec<Result<Vec<u8>, TransportError>> {
1783 let workers = (0..fetches.len().min(4))
1792 .map(|_| {
1793 let mut worker = self.fork_round();
1794 worker.round_deadline_at = self.round_deadline_at;
1795 worker
1796 })
1797 .collect::<Vec<_>>();
1798 let slots = fetches.iter().map(|_| Mutex::new(None)).collect::<Vec<_>>();
1799 let progress = Mutex::new(on_progress);
1800 let next = std::sync::atomic::AtomicUsize::new(0);
1801 std::thread::scope(|scope| {
1802 for worker in workers {
1803 let (slots, progress, next) = (&slots, &progress, &next);
1804 scope.spawn(move || {
1805 let mut worker = worker;
1806 loop {
1807 let index = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1808 let Some(fetch) = fetches.get(index) else {
1809 break;
1810 };
1811 let mut report = |bytes: u64| {
1812 (*progress.lock().expect("segment progress lock"))(index, bytes);
1813 };
1814 let result = match fetch {
1815 SegmentFetch::Direct(request) => {
1816 worker.download_segment(request, &mut report)
1817 }
1818 SegmentFetch::Url(url) => worker.fetch_url(url, &mut report),
1819 };
1820 *slots[index].lock().expect("segment result slot") = Some(result);
1821 }
1822 });
1823 }
1824 });
1825 slots
1826 .into_iter()
1827 .map(|slot| slot.into_inner().expect("segment result slot"))
1828 .map(|result| result.expect("every fetch is attempted"))
1829 .collect()
1830 }
1831
1832 fn fetch_url(
1833 &mut self,
1834 url: &str,
1835 on_progress: &mut dyn FnMut(u64),
1836 ) -> Result<Vec<u8>, TransportError> {
1837 let req = self.finish_request(self.agent.get(url), true)?;
1839 let resp = req
1840 .call()
1841 .map_err(|error| segment_error(http_err("segment request failed", error)))?;
1842 read_body(resp, Some(on_progress), self.policy.max_response_bytes)
1843 .map_err(segment_error)
1844 }
1845
1846 fn blob_upload(
1847 &mut self,
1848 blob_id: &str,
1849 bytes: &[u8],
1850 media_type: Option<&str>,
1851 ) -> Result<(), TransportError> {
1852 self.check_request_bytes(bytes.len() as u64)?;
1855 let credentialed = self.credentialed();
1856 let url = format!("{}/blobs/{}", self.base_url, blob_id);
1857 let mut req = self.agent.put(&url).header(
1858 "content-type",
1859 media_type.unwrap_or("application/octet-stream"),
1860 );
1861 for (k, v) in &self.headers {
1862 req = req.header(k.as_str(), v.as_str());
1863 }
1864 let req = self.finish_request(req, credentialed)?;
1865 let resp = req
1866 .send(bytes)
1867 .map_err(|e| http_err("blob upload failed", e))?;
1868 read_body(resp, None, self.policy.max_response_bytes)?;
1869 Ok(())
1870 }
1871
1872 fn blob_download(&mut self, blob_id: &str) -> Result<BlobDownload, TransportError> {
1873 let credentialed = self.credentialed();
1876 let url = format!("{}/blobs/{}", self.base_url, blob_id);
1877 let mut req = self.agent.get(&url);
1878 for (k, v) in &self.headers {
1879 req = req.header(k.as_str(), v.as_str());
1880 }
1881 let req = self.finish_request(req, credentialed)?;
1882 let resp = req.call().map_err(|cause| {
1883 let status = match &cause {
1884 ureq::Error::StatusCode(status) => Some(*status),
1885 _ => None,
1886 };
1887 let mut error = http_err("blob download failed", cause);
1888 match status {
1889 Some(404) => {
1890 error.code = "blob.not_found".into();
1891 error.message = "blob not found".into();
1892 }
1893 Some(403) => {
1894 error.code = "blob.forbidden".into();
1895 error.message = "blob download forbidden".into();
1896 }
1897 Some(401) => {
1898 error.code = "sync.auth_required".into();
1899 error.message = "host authentication required".into();
1900 }
1901 _ => {}
1902 }
1903 error
1904 })?;
1905 let is_json = resp
1908 .headers()
1909 .get(ureq::http::header::CONTENT_TYPE)
1910 .and_then(|value| value.to_str().ok())
1911 .is_some_and(|value| value.contains("application/json"));
1912 let body =
1913 read_body(resp, None, self.policy.max_response_bytes).map_err(|mut error| {
1914 let status = error
1918 .details
1919 .as_ref()
1920 .filter(|details| {
1921 details.get("causeKind").and_then(serde_json::Value::as_str)
1922 == Some("status")
1923 })
1924 .and_then(|details| details.get("httpStatus"))
1925 .and_then(serde_json::Value::as_u64);
1926 match status {
1927 Some(404) => {
1928 error.code = "blob.not_found".into();
1929 error.message = "blob not found".into();
1930 }
1931 Some(403) => {
1932 error.code = "blob.forbidden".into();
1933 error.message = "blob download forbidden".into();
1934 }
1935 Some(401) => {
1936 error.code = "sync.auth_required".into();
1937 error.message = "host authentication required".into();
1938 }
1939 Some(_) => {
1940 error.code = "transport.failed".into();
1941 error.message = "blob download failed".into();
1942 }
1943 None => {}
1944 }
1945 error
1946 })?;
1947 if is_json {
1948 if let Ok(parsed) = serde_json::from_slice::<serde_json::Value>(&body) {
1949 if let Some(u) = parsed.get("url").and_then(|v| v.as_str()) {
1950 return Ok(BlobDownload::Url {
1951 url: u.to_owned(),
1952 url_expires_at_ms: parsed
1953 .get("urlExpiresAtMs")
1954 .and_then(|v| v.as_i64()),
1955 });
1956 }
1957 }
1958 }
1959 Ok(BlobDownload::Bytes(body))
1960 }
1961
1962 fn fetch_blob_url(&mut self, url: &str) -> Result<Vec<u8>, TransportError> {
1963 self.get_bytes(url, false)
1965 }
1966
1967 fn blob_upload_grant(
1968 &mut self,
1969 blob_id: &str,
1970 byte_length: u64,
1971 media_type: Option<&str>,
1972 ) -> Result<BlobUploadGrant, TransportError> {
1973 let url = format!("{}/blobs/{}/upload-grant", self.base_url, blob_id);
1974 let body = serde_json::json!({
1975 "byteLength": byte_length,
1976 "mediaType": media_type,
1977 })
1978 .to_string();
1979 self.check_request_bytes(body.len() as u64)?;
1980 let credentialed = self.credentialed();
1981 let mut req = self
1982 .agent
1983 .post(&url)
1984 .header("content-type", "application/json");
1985 for (k, v) in &self.headers {
1986 req = req.header(k.as_str(), v.as_str());
1987 }
1988 let req = self.finish_request(req, credentialed)?;
1989 let resp = req
1990 .send(body)
1991 .map_err(|e| http_err("blob upload grant request failed", e))?;
1992 let grant_body = read_body(resp, None, self.policy.max_response_bytes)?;
1993 let parsed: serde_json::Value = serde_json::from_slice(&grant_body)
1994 .map_err(|_| transfer_error("blob upload grant decode failed", "decode", None))?;
1995 if let Some(u) = parsed.get("url").and_then(|v| v.as_str()) {
1996 return Ok(BlobUploadGrant::Url {
1997 url: u.to_owned(),
1998 url_expires_at_ms: parsed.get("urlExpiresAtMs").and_then(|v| v.as_i64()),
1999 });
2000 }
2001 if parsed.get("present").and_then(|v| v.as_bool()) == Some(true) {
2002 return Ok(BlobUploadGrant::Present);
2003 }
2004 Ok(BlobUploadGrant::None)
2005 }
2006
2007 fn blob_put_url(
2008 &mut self,
2009 url: &str,
2010 bytes: &[u8],
2011 media_type: Option<&str>,
2012 ) -> Result<(), TransportError> {
2013 self.check_request_bytes(bytes.len() as u64)?;
2015 let req = self.finish_request(
2016 self.agent.put(url).header(
2017 "content-type",
2018 media_type.unwrap_or("application/octet-stream"),
2019 ),
2020 true,
2021 )?;
2022 let resp = req
2023 .send(bytes)
2024 .map_err(|e| http_err("signed blob upload failed", e))?;
2025 read_body(resp, None, self.policy.max_response_bytes)?;
2026 Ok(())
2027 }
2028
2029 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2030 if self.outgoing.is_some() {
2031 if self.reader_stop.load(Ordering::SeqCst)
2032 || self
2033 .reader
2034 .as_ref()
2035 .is_some_and(|reader| reader.is_finished())
2036 {
2037 self.shutdown();
2038 } else {
2039 return Ok(());
2040 }
2041 }
2042 use tungstenite::client::IntoClientRequest;
2048 let mut url = url::Url::parse(&self.ws_url)
2049 .map_err(|_| transfer_error("realtime URL invalid", "request", None))?;
2050 if let Some(client_id) = self.realtime_client_id.as_deref() {
2051 let retained_query: Vec<(String, String)> = url
2052 .query_pairs()
2053 .filter(|(key, _)| key != "clientId")
2054 .map(|(key, value)| (key.into_owned(), value.into_owned()))
2055 .collect();
2056 url.set_query(None);
2057 url.query_pairs_mut()
2058 .extend_pairs(retained_query)
2059 .append_pair("clientId", client_id);
2060 }
2061 let mut request = url
2062 .as_str()
2063 .into_client_request()
2064 .map_err(|error| ws_err("realtime request failed", error))?;
2065 {
2066 let out = request.headers_mut();
2067 for (k, v) in &self.headers {
2068 if let (Ok(name), Ok(value)) = (
2069 tungstenite::http::HeaderName::try_from(k.as_str()),
2070 tungstenite::http::HeaderValue::try_from(v.as_str()),
2071 ) {
2072 out.insert(name, value);
2073 }
2074 }
2075 }
2076 if self.policy.redirects == RedirectPolicy::Follow {
2079 return Err(redirect_error(
2080 "realtime establishment does not follow redirects",
2081 None,
2082 ));
2083 }
2084 let mut ws = connect_websocket(request, Instant::now() + CONNECT_TIMEOUT)?;
2085 let stream = match ws.get_mut() {
2086 MaybeTlsStream::Plain(stream) => stream.established(),
2087 MaybeTlsStream::Rustls(stream) => stream.get_mut().established(),
2088 _ => {
2089 return Err(TransportError::new(
2090 "transport.failed",
2091 "unsupported websocket stream",
2092 ))
2093 }
2094 }
2095 .map_err(|e| io_err("realtime clone failed", e))?;
2096 stream
2097 .set_nonblocking(true)
2098 .map_err(|e| io_err("realtime nonblocking failed", e))?;
2099 let poller = Arc::new(Poller::new().map_err(|e| io_err("realtime poller failed", e))?);
2100 unsafe { poller.add(&stream, Event::readable(0)) }
2102 .map_err(|e| io_err("realtime register failed", e))?;
2103 let registration = SocketRegistration {
2104 poller: Arc::clone(&poller),
2105 stream,
2106 };
2107 let (outgoing, queued) = mpsc::sync_channel::<Outgoing>(1);
2108 self.outgoing = Some(Box::new(SocketSender {
2109 queue: outgoing,
2110 poller,
2111 }));
2112 self.reader_stop = Arc::new(AtomicBool::new(false));
2113 self.round = Arc::new(RoundChannel::default());
2114 let inbound = Arc::clone(&self.inbound);
2115 let stop = Arc::clone(&self.reader_stop);
2116 let round = Arc::clone(&self.round);
2117 self.reader = Some(std::thread::spawn(move || {
2118 let mut events = Events::new();
2119 let mut pending: Option<mpsc::Sender<Result<(), TransportError>>> = None;
2120 let failure = 'io: loop {
2121 if stop.load(Ordering::SeqCst) {
2122 let _ = ws.close(None);
2123 let _ = ws.flush();
2124 break TransportError::new(
2125 "sync.transport_failed",
2126 "realtime disconnected mid-round (§8.7)",
2127 );
2128 }
2129 if pending.is_none() {
2130 if let Ok(outgoing) = queued.try_recv() {
2131 pending = Some(outgoing.completed);
2132 if let Err(error) = ws.write(outgoing.message) {
2134 if !is_would_block(&error) {
2135 break ws_err("realtime write failed", error);
2136 }
2137 }
2138 }
2139 }
2140 let mut drained = false;
2142 for _ in 0..64 {
2143 match ws.read() {
2144 Ok(Message::Text(text)) => {
2145 inbound.push(Inbound::Text(text.to_string()))
2146 }
2147 Ok(Message::Binary(bytes)) => {
2148 if let Some(delta) = round.route_binary(&bytes) {
2149 inbound.push(Inbound::Binary(delta));
2150 }
2151 }
2152 Err(error) if is_would_block(&error) => {
2153 drained = true;
2154 break;
2155 }
2156 Ok(Message::Close(_)) => {
2157 let _ = ws.flush();
2158 break 'io TransportError::new(
2159 "sync.transport_failed",
2160 "realtime disconnected mid-round (§8.7)",
2161 );
2162 }
2163 Err(_) => {
2164 break 'io TransportError::new(
2165 "sync.transport_failed",
2166 "realtime disconnected mid-round (§8.7)",
2167 );
2168 }
2169 Ok(_) => {}
2170 }
2171 }
2172 let interest = match ws.flush() {
2173 Ok(()) => {
2174 if let Some(completed) = pending.take() {
2175 let _ = completed.send(Ok(()));
2176 }
2177 Event::readable(0)
2178 }
2179 Err(error) if is_would_block(&error) => Event::all(0),
2180 Err(error) => break ws_err("realtime flush failed", error),
2181 };
2182 if !drained {
2183 continue;
2184 }
2185 if let Err(error) = registration.poller.modify(®istration.stream, interest) {
2186 break io_err("realtime rearm failed", error);
2187 }
2188 events.clear();
2189 if let Err(error) = registration.poller.wait(&mut events, None) {
2190 if error.kind() != std::io::ErrorKind::Interrupted {
2191 break io_err("realtime wait failed", error);
2192 }
2193 }
2194 };
2195 if !stop.load(Ordering::SeqCst) {
2200 inbound.push(Inbound::Disconnected(failure.clone()));
2201 }
2202 if let Some(completed) = pending {
2203 let _ = completed.send(Err(failure.clone()));
2204 }
2205 while let Ok(outgoing) = queued.try_recv() {
2206 let _ = outgoing.completed.send(Err(failure.clone()));
2207 }
2208 round.fail_in_flight(failure);
2209 }));
2210 Ok(())
2211 }
2212
2213 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
2214 self.send_message(Message::Text(text.to_owned().into()))
2215 }
2216
2217 fn realtime_close(&mut self) -> Result<(), TransportError> {
2218 self.shutdown();
2219 Ok(())
2220 }
2221
2222 fn round_deadline(&self) -> Option<Duration> {
2223 NativeTransport::round_deadline(self)
2224 }
2225
2226 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
2227 NativeTransport::set_round_deadline(self, deadline);
2228 }
2229 }
2230 #[cfg(test)]
2231 mod transfer_tests {
2232 use super::*;
2233 use ureq::unversioned::resolver::DefaultResolver;
2234 use ureq::unversioned::transport::{ConnectionDetails, Connector};
2235
2236 const SECRET: &str = "https://user:password@cdn.example/secret-capability-path?secret-query=value#secret-fragment";
2237
2238 #[derive(Debug)]
2239 struct SecretError;
2240 impl std::fmt::Display for SecretError {
2241 fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2242 panic!("transport error must never be formatted")
2243 }
2244 }
2245 impl std::error::Error for SecretError {}
2246
2247 #[derive(Debug)]
2248 struct FailingConnector(&'static str);
2249 impl Connector for FailingConnector {
2250 type Out = ();
2251 fn connect(
2252 &self,
2253 _: &ConnectionDetails,
2254 _: Option<()>,
2255 ) -> Result<Option<()>, ureq::Error> {
2256 Err(match self.0 {
2257 "io" => ureq::Error::Io(std::io::Error::new(
2258 std::io::ErrorKind::ConnectionRefused,
2259 format!("404 {SECRET}"),
2260 )),
2261 "tls" => ureq::Error::Tls(SECRET),
2262 "request" => ureq::Error::BadUri(SECRET.into()),
2263 "timeout" => ureq::Error::Timeout(ureq::Timeout::Global),
2264 "404" => ureq::Error::StatusCode(404),
2265 "401" => ureq::Error::StatusCode(401),
2266 "403" => ureq::Error::StatusCode(403),
2267 "503" => ureq::Error::StatusCode(503),
2268 _ => ureq::Error::Other(Box::new(SecretError)),
2269 })
2270 }
2271 }
2272
2273 #[test]
2274 fn injected_http_failures_never_expose_capabilities_or_format_the_cause() {
2275 for kind in [
2276 "io", "tls", "request", "timeout", "unknown", "404", "401", "403", "503",
2277 ] {
2278 for operation in [
2279 "sync",
2280 "remote",
2281 "segment",
2282 "signed_segment",
2283 "blob",
2284 "signed_blob",
2285 "upload",
2286 "grant",
2287 "signed_put",
2288 ] {
2289 let mut transport =
2290 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None)
2291 .unwrap();
2292 transport.agent = ureq::Agent::with_parts(
2293 ureq::Agent::config_builder().build(),
2294 FailingConnector(kind),
2295 DefaultResolver::default(),
2296 );
2297 let mut progress = |_| {};
2298 let result = match operation {
2299 "sync" => transport.sync(&[]).map(|_| ()),
2300 "remote" => transport.remote_operation(&[]).map(|_| ()),
2301 "segment" => transport.download_segment(&SegmentRequest { segment_id: "sha256:test".into(), table: "tasks".into(), requested_scopes_json: "{}".into() }, &mut progress).map(|_| ()),
2302 "signed_segment" => transport.fetch_url("http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment", &mut progress).map(|_| ()),
2303 "blob" => transport.blob_download("sha256:test").map(|_| ()),
2304 "signed_blob" => transport.fetch_blob_url("http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment").map(|_| ()),
2305 "upload" => transport.blob_upload("sha256:test", &[], None),
2306 "grant" => transport.blob_upload_grant("sha256:test", 0, None).map(|_| ()),
2307 _ => transport.blob_put_url("http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment", &[], None),
2308 };
2309 let error = result.unwrap_err();
2310 let expected_code = match (operation, kind) {
2311 (_, "timeout") => "transport.timeout",
2312 ("segment" | "signed_segment", _) => "sync.transport_failed",
2313 ("blob", "404") => "blob.not_found",
2314 ("blob", "401") => "sync.auth_required",
2315 ("blob", "403") => "blob.forbidden",
2316 _ => "transport.failed",
2317 };
2318 assert_eq!(error.code, expected_code, "{operation} {kind}");
2319 let details = error.details.unwrap();
2320 assert_eq!(
2321 details["causeKind"],
2322 match kind {
2323 "io" => "connect",
2324 "404" | "401" | "403" | "503" => "status",
2325 other => other,
2326 }
2327 );
2328 assert_eq!(
2329 details.get("httpStatus").and_then(|value| value.as_u64()),
2330 kind.parse::<u64>().ok()
2331 );
2332 let rendered = format!("{}{}", error.message, details);
2333 for secret in [
2334 "http://",
2335 "https://",
2336 "password",
2337 "secret-capability-path",
2338 "secret-query",
2339 "secret-fragment",
2340 ] {
2341 assert!(!rendered.contains(secret), "{operation} {kind}");
2342 }
2343 assert!(details.get("causeMessage").is_none());
2344 assert!(details.get("path").is_none());
2345 }
2346 }
2347 }
2348
2349 #[test]
2350 fn injected_body_and_websocket_errors_do_not_expose_payloads_or_source_chains() {
2351 struct FailingRead;
2352 impl std::io::Read for FailingRead {
2353 fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
2354 Err(std::io::Error::other(SECRET))
2355 }
2356 }
2357 let response = ureq::http::Response::builder()
2358 .status(200)
2359 .body(ureq::Body::builder().reader(FailingRead))
2360 .unwrap();
2361 let error = segment_error(read_body(response, None, None).unwrap_err());
2362 assert_eq!(error.code, "sync.transport_failed");
2363 assert_eq!(error.message, "segment transfer failed");
2364 assert_eq!(
2365 error.details,
2366 Some(serde_json::json!({ "causeKind": "body", "httpStatus": 200 }))
2367 );
2368 for cause in [
2369 tungstenite::Error::Io(std::io::Error::other(SecretError)),
2370 tungstenite::Error::Http(Box::new(
2371 tungstenite::http::Response::builder()
2372 .status(403)
2373 .header("location", SECRET)
2374 .body(Some(SECRET.as_bytes().to_vec()))
2375 .unwrap(),
2376 )),
2377 ] {
2378 let error = ws_err("realtime connection failed", cause);
2379 assert_eq!(error.message, "realtime connection failed");
2380 let details = error.details.unwrap();
2381 assert!(details.get("path").is_none());
2382 assert!(details.get("causeMessage").is_none());
2383 assert!(!details.to_string().contains("secret"));
2384 }
2385 }
2386
2387 #[test]
2388 fn body_read_timeouts_are_classified_through_ureq_wrapping() {
2389 let direct = ureq::Error::Timeout(ureq::Timeout::Global).into_io();
2391 assert!(is_timeout_io(&direct));
2392 let wrapped = ureq::Error::Decompress(
2394 "gzip",
2395 ureq::Error::Timeout(ureq::Timeout::Global).into_io(),
2396 )
2397 .into_io();
2398 assert!(is_timeout_io(&wrapped));
2399 assert!(is_timeout_io(&std::io::Error::new(
2401 std::io::ErrorKind::TimedOut,
2402 "plain"
2403 )));
2404 assert!(!is_timeout_io(&std::io::Error::other("other")));
2405 assert!(!is_timeout_io(&ureq::Error::StatusCode(500).into_io()));
2406 }
2407
2408 #[test]
2409 fn body_read_wrapped_timeouts_surface_the_typed_code() {
2410 struct TimeoutRead(bool);
2411 impl std::io::Read for TimeoutRead {
2412 fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
2413 Err(if self.0 {
2414 ureq::Error::Timeout(ureq::Timeout::Global).into_io()
2415 } else {
2416 ureq::Error::Decompress(
2417 "gzip",
2418 ureq::Error::Timeout(ureq::Timeout::Global).into_io(),
2419 )
2420 .into_io()
2421 })
2422 }
2423 }
2424 for compressed in [false, true] {
2425 let response = ureq::http::Response::builder()
2426 .status(200)
2427 .body(ureq::Body::builder().reader(TimeoutRead(compressed)))
2428 .unwrap();
2429 let error = read_body(response, None, None).unwrap_err();
2430 assert_eq!(error.code, "transport.timeout", "{compressed}");
2431 assert_eq!(
2432 error.details,
2433 Some(serde_json::json!({ "causeKind": "timeout" }))
2434 );
2435 }
2436 }
2437
2438 #[test]
2439 fn policy_validation_rejects_zero_and_unrepresentable_bounds() {
2440 assert_eq!(
2441 HostTransportPolicy {
2442 request_timeout: Some(Duration::ZERO),
2443 ..Default::default()
2444 }
2445 .validate()
2446 .unwrap_err(),
2447 "sync.invalid_request: transport config requestTimeoutMs is out of range"
2448 );
2449 assert_eq!(
2450 HostTransportPolicy {
2451 round_deadline: Some(Duration::MAX),
2452 ..Default::default()
2453 }
2454 .validate()
2455 .unwrap_err(),
2456 "sync.invalid_request: transport config roundDeadlineMs is out of range"
2457 );
2458 assert!(HostTransportPolicy {
2459 max_request_bytes: Some(0),
2460 ..Default::default()
2461 }
2462 .validate()
2463 .is_err());
2464 assert!(HostTransportPolicy {
2465 max_response_bytes: Some(0),
2466 ..Default::default()
2467 }
2468 .validate()
2469 .is_err());
2470 assert!(HostTransportPolicy::default().validate().is_ok());
2473 assert_eq!(
2474 HostTransportPolicy::from_config(&serde_json::json!({ "requestTimeoutMs": 0 }))
2475 .unwrap_err(),
2476 "sync.invalid_request: transport config requestTimeoutMs must be a positive integer"
2477 );
2478 }
2479
2480 #[test]
2481 fn set_policy_rejects_invalid_and_preserves_the_current_policy() {
2482 let mut transport =
2483 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None).unwrap();
2484 assert_eq!(transport.policy.max_response_bytes, None);
2485 let error = transport
2486 .set_policy(HostTransportPolicy {
2487 max_response_bytes: Some(0),
2488 ..Default::default()
2489 })
2490 .unwrap_err();
2491 assert_eq!(
2492 error,
2493 "sync.invalid_request: transport config maxResponseBytes must be a positive integer"
2494 );
2495 assert_eq!(transport.policy.max_response_bytes, None);
2496 }
2497
2498 #[test]
2499 fn an_expired_round_deadline_is_typed_and_cleared_for_the_next_round() {
2500 let mut transport =
2501 NativeTransport::new("http://127.0.0.1:9", &serde_json::json!({}), None).unwrap();
2502 transport.set_round_deadline(Some(std::time::Instant::now() - Duration::from_secs(1)));
2503 let error = transport.blob_download("sha256:test").unwrap_err();
2504 assert_eq!(error.code, "transport.timeout");
2505 assert_eq!(
2506 error.details,
2507 Some(serde_json::json!({ "causeKind": "round_deadline" }))
2508 );
2509 transport.set_round_deadline(None);
2511 assert!(transport.round_remaining().is_ok());
2512 }
2513
2514 #[test]
2515 fn an_expired_round_deadline_refuses_before_queuing_a_realtime_send() {
2516 let mut transport =
2517 NativeTransport::new("http://127.0.0.1:9", &serde_json::json!({}), None).unwrap();
2518 transport.set_round_deadline(Some(std::time::Instant::now() - Duration::from_secs(1)));
2519 let error = transport.realtime_send("blocked").unwrap_err();
2523 assert_eq!(error.code, "transport.timeout");
2524 assert_eq!(
2525 error.details,
2526 Some(serde_json::json!({ "causeKind": "round_deadline" }))
2527 );
2528 transport.set_round_deadline(None);
2531 let error = transport.realtime_send("blocked").unwrap_err();
2532 assert_eq!(error.code, "transport.failed");
2533 }
2534
2535 #[test]
2536 fn the_request_timeout_bounds_phases_and_the_round_deadline_bounds_the_call() {
2537 #[derive(Debug)]
2538 struct TimeoutRecording(Arc<Mutex<Vec<Option<Duration>>>>);
2539 impl Connector for TimeoutRecording {
2540 type Out = ();
2541 fn connect(
2542 &self,
2543 details: &ConnectionDetails,
2544 _: Option<()>,
2545 ) -> Result<Option<()>, ureq::Error> {
2546 self.0
2547 .lock()
2548 .unwrap()
2549 .push(details.config.timeouts().global);
2550 Err(ureq::Error::ConnectionFailed)
2551 }
2552 }
2553 let mut transport =
2554 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None).unwrap();
2555 transport
2556 .set_policy(HostTransportPolicy {
2557 request_timeout: Some(Duration::from_millis(1234)),
2558 ..Default::default()
2559 })
2560 .unwrap();
2561 let timeouts = transport.agent.config().timeouts();
2562 for phase in [
2563 timeouts.resolve,
2564 timeouts.connect,
2565 timeouts.send_request,
2566 timeouts.recv_response,
2567 ] {
2568 assert_eq!(phase, Some(Duration::from_millis(1234)));
2569 }
2570 assert_eq!(timeouts.global, None, "the request bound is not end to end");
2571 assert_eq!(timeouts.recv_body, None, "a body is bounded by silence");
2572 let seen = Arc::new(Mutex::new(Vec::new()));
2573 transport.agent = ureq::Agent::with_parts(
2574 ureq::Agent::config_builder().build(),
2575 TimeoutRecording(Arc::clone(&seen)),
2576 DefaultResolver::default(),
2577 );
2578 let _ = transport.blob_upload("sha256:test", &[0u8; 4], None);
2579 transport.set_round_deadline(Some(std::time::Instant::now() + Duration::from_secs(60)));
2580 let _ = transport.blob_upload("sha256:test", &[0u8; 4], None);
2581 let seen = seen.lock().unwrap();
2582 assert_eq!(seen[0], None, "no round deadline, no call ceiling");
2583 assert!(seen[1].is_some_and(|ceiling| ceiling <= Duration::from_secs(60)));
2584 }
2585
2586 #[test]
2587 fn the_idle_bound_replaces_an_unbounded_or_longer_wait_and_keeps_a_shorter_one() {
2588 let idle = Duration::from_secs(30);
2589 let unbounded = NextTimeout {
2590 after: UreqDuration::NotHappening,
2591 reason: UreqTimeout::Global,
2592 };
2593 assert_eq!(idle_bound(unbounded, idle).after, UreqDuration::Exact(idle));
2594 let longer = NextTimeout {
2595 after: UreqDuration::from_secs(90),
2596 reason: UreqTimeout::Global,
2597 };
2598 assert_eq!(idle_bound(longer, idle).after, UreqDuration::Exact(idle));
2599 let shorter = NextTimeout {
2600 after: UreqDuration::from_secs(5),
2601 reason: UreqTimeout::Global,
2602 };
2603 assert_eq!(idle_bound(shorter, idle), shorter);
2604 }
2605
2606 fn shaped_link(
2611 total: u64,
2612 chunk: u64,
2613 latency: Duration,
2614 rate: u64,
2615 stalls: &[(Duration, Duration)],
2616 ) -> Vec<Duration> {
2617 let mut at = latency;
2618 let mut arrivals = Vec::new();
2619 let mut left = total;
2620 while left > 0 {
2621 let piece = left.min(chunk);
2622 at += Duration::from_micros(piece * 1_000_000 / rate);
2623 for (start, length) in stalls {
2624 if at > *start && at < *start + *length {
2625 at = *start + *length;
2626 }
2627 }
2628 arrivals.push(at);
2629 left -= piece;
2630 }
2631 arrivals
2632 }
2633
2634 fn round_over(
2638 arrivals: &[Duration],
2639 idle: Duration,
2640 ceiling: Option<Duration>,
2641 ) -> Result<Duration, (Duration, bool)> {
2642 let base = Instant::now();
2643 let mut last = base;
2644 for arrival in arrivals {
2645 let (deadline, ceiling_binds) =
2646 round_expiry(last, idle, ceiling.map(|ceiling| base + ceiling));
2647 if base + *arrival > deadline {
2648 return Err((deadline - base, ceiling_binds));
2649 }
2650 last = base + *arrival;
2651 }
2652 Ok(last - base)
2653 }
2654
2655 #[test]
2656 fn a_slow_link_that_keeps_delivering_never_trips_the_idle_bound() {
2657 let link = shaped_link(
2659 8_600_000,
2660 64 * 1024,
2661 Duration::from_millis(80),
2662 100_000,
2663 &[],
2664 );
2665 let finished = round_over(&link, ROUND_IDLE_TIMEOUT, None).unwrap();
2666 assert!(
2667 finished > ROUND_IDLE_TIMEOUT * 2,
2668 "the transfer outlasts the old whole-transfer bound ({finished:?})"
2669 );
2670 }
2671
2672 #[test]
2673 fn a_stall_shorter_than_the_idle_bound_is_survived_and_a_longer_one_is_not() {
2674 let link = |length| {
2675 shaped_link(
2676 4_000_000,
2677 64 * 1024,
2678 Duration::from_millis(80),
2679 100_000,
2680 &[(Duration::from_secs(10), length)],
2681 )
2682 };
2683 round_over(&link(Duration::from_secs(29)), ROUND_IDLE_TIMEOUT, None).unwrap();
2684 let (expired, ceiling_bound) =
2685 round_over(&link(Duration::from_secs(31)), ROUND_IDLE_TIMEOUT, None).unwrap_err();
2686 assert!(!ceiling_bound);
2687 assert!(
2688 expired >= Duration::from_secs(30) && expired < Duration::from_secs(41),
2689 "the bound counts from the last chunk before the stall ({expired:?})"
2690 );
2691 }
2692
2693 #[test]
2694 fn the_host_ceiling_still_bounds_a_link_that_keeps_delivering() {
2695 let link = shaped_link(
2696 8_600_000,
2697 64 * 1024,
2698 Duration::from_millis(80),
2699 100_000,
2700 &[],
2701 );
2702 let (expired, ceiling_bound) =
2703 round_over(&link, ROUND_IDLE_TIMEOUT, Some(Duration::from_secs(60))).unwrap_err();
2704 assert!(ceiling_bound);
2705 assert_eq!(expired, Duration::from_secs(60));
2706 }
2707
2708 #[test]
2709 fn an_expired_round_deadline_bounds_the_socket_wait() {
2710 let channel = RoundChannel::default();
2711 let error = channel
2712 .wait(
2713 Some(std::time::Instant::now() - Duration::from_secs(1)),
2714 ROUND_IDLE_TIMEOUT,
2715 )
2716 .unwrap_err();
2717 assert_eq!(error.code, "transport.timeout");
2718 assert_eq!(
2719 error.details,
2720 Some(serde_json::json!({ "causeKind": "round_deadline" }))
2721 );
2722 }
2723 }
2724}
2725
2726#[cfg(all(test, feature = "native-transport"))]
2727mod tests {
2728 use super::*;
2729 use std::io::{BufRead, BufReader, Write};
2730
2731 #[test]
2732 fn detached_round_uses_the_owned_socket_and_shutdown_releases_it() {
2733 use ssp2::model::{Frame, Message, MsgKind};
2734 use std::sync::mpsc::channel;
2735 for cancelled in [false, true] {
2736 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2737 let base = format!("http://{}", listener.local_addr().unwrap());
2738 let (entered_tx, entered_rx) = channel();
2739 let (release_tx, release_rx) = channel();
2740 let server = std::thread::spawn(move || {
2741 let (socket, _) = listener.accept().unwrap();
2742 let mut socket = tungstenite::accept(socket).unwrap();
2743 let request = socket.read().unwrap().into_data();
2744 assert_eq!(request[0], crate::REALTIME_TAG_ROUND);
2745 assert_eq!(
2746 ssp2::decode_message(&request[1..]).unwrap().msg_kind,
2747 MsgKind::Request
2748 );
2749 entered_tx.send(()).unwrap();
2750 release_rx.recv().unwrap();
2751 if !cancelled {
2752 let response = ssp2::encode_message(&Message {
2753 wire_version: ssp2::decode::WIRE_VERSION,
2754 msg_kind: MsgKind::Response,
2755 frames: vec![Frame::RespHeader {
2756 required_schema_version: None,
2757 latest_schema_version: None,
2758 log_epoch: Some("epoch".into()),
2759 reset_required: Some(true),
2760 }],
2761 });
2762 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
2763 chunk.extend(response);
2764 socket
2765 .send(tungstenite::Message::Binary(chunk.into()))
2766 .unwrap();
2767 }
2768 });
2769 let mut owner =
2770 HostTransport::new_from_config(&serde_json::json!({"baseUrl":base})).unwrap();
2771 owner.realtime_connect_for_client("detached-round").unwrap();
2772 let mut detached = owner.fork_round();
2773 let network = std::thread::spawn(move || {
2774 let request = ssp2::encode_message(&Message {
2775 wire_version: ssp2::decode::WIRE_VERSION,
2776 msg_kind: MsgKind::Request,
2777 frames: vec![
2778 Frame::ReqHeader {
2779 client_id: "detached-round".into(),
2780 schema_version: 1,
2781 log_epoch: None,
2782 },
2783 Frame::PullHeader {
2784 limit_commits: 0,
2785 limit_snapshot_rows: 0,
2786 max_snapshot_pages: 0,
2787 accept: 3,
2788 },
2789 ],
2790 });
2791 detached.realtime_sync(&request)
2792 });
2793 entered_rx.recv().unwrap();
2794 owner.set_headers(vec![("authorization".into(), "Bearer rotated".into())]);
2795 assert!(owner.take_inbound().is_empty());
2796 if cancelled {
2797 owner.shutdown();
2798 }
2799 release_tx.send(()).unwrap();
2800 let response = network.join().unwrap();
2801 if cancelled {
2802 assert!(response.is_err());
2803 } else {
2804 assert_eq!(
2805 ssp2::decode_message(&response.unwrap()).unwrap().msg_kind,
2806 MsgKind::Response
2807 );
2808 owner.shutdown();
2809 }
2810 server.join().unwrap();
2811 }
2812 }
2813
2814 fn minimal_response() -> Vec<u8> {
2815 use ssp2::model::{Frame, Message, MsgKind};
2816 ssp2::encode_message(&Message {
2817 wire_version: ssp2::decode::WIRE_VERSION,
2818 msg_kind: MsgKind::Response,
2819 frames: vec![Frame::RespHeader {
2820 required_schema_version: None,
2821 latest_schema_version: None,
2822 log_epoch: Some("epoch".into()),
2823 reset_required: Some(true),
2824 }],
2825 })
2826 }
2827
2828 #[test]
2829 fn a_stalled_response_body_times_out_on_silence() {
2830 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2831 let base = format!("http://{}", listener.local_addr().unwrap());
2832 let server = std::thread::spawn(move || {
2833 let (mut socket, _) = listener.accept().unwrap();
2834 let mut reader = BufReader::new(socket.try_clone().unwrap());
2835 loop {
2836 let mut line = String::new();
2837 reader.read_line(&mut line).unwrap();
2838 if line == "\r\n" {
2839 break;
2840 }
2841 }
2842 socket
2843 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n0123456789")
2844 .unwrap();
2845 let mut rest = Vec::new();
2847 let _ = std::io::Read::read_to_end(&mut reader, &mut rest);
2848 });
2849 let mut transport = HostTransport::new_from_config(
2850 &serde_json::json!({"baseUrl": base, "requestTimeoutMs": 200}),
2851 )
2852 .unwrap();
2853 let mut received = 0;
2854 let error = transport
2855 .download_segment(
2856 &SegmentRequest {
2857 segment_id: "sha256:test".into(),
2858 table: "tasks".into(),
2859 requested_scopes_json: "{}".into(),
2860 },
2861 &mut |bytes| received = bytes,
2862 )
2863 .unwrap_err();
2864 assert_eq!(error.code, "transport.timeout");
2865 assert_eq!(
2866 error.details,
2867 Some(serde_json::json!({ "causeKind": "timeout" }))
2868 );
2869 server.join().unwrap();
2870 }
2871
2872 #[test]
2873 fn a_timed_out_socket_round_closes_the_socket_and_reports_the_loss() {
2874 use std::sync::mpsc::channel;
2875 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2876 let base = format!("http://{}", listener.local_addr().unwrap());
2877 let (closed_tx, closed_rx) = channel();
2878 let server = std::thread::spawn(move || {
2879 let (socket, _) = listener.accept().unwrap();
2880 let mut socket = tungstenite::accept(socket).unwrap();
2881 socket.read().unwrap();
2882 let response = minimal_response();
2884 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
2885 chunk.extend_from_slice(&response[..response.len() / 2]);
2886 socket
2887 .send(tungstenite::Message::Binary(chunk.into()))
2888 .unwrap();
2889 while let Ok(message) = socket.read() {
2891 if matches!(message, tungstenite::Message::Close(_)) {
2892 break;
2893 }
2894 }
2895 closed_tx.send(()).unwrap();
2896 });
2897 let mut transport =
2898 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
2899 transport.realtime_connect_for_client("slow").unwrap();
2900 transport.set_round_deadline(Some(
2901 std::time::Instant::now() + std::time::Duration::from_millis(300),
2902 ));
2903 let error = transport.realtime_sync(b"request").unwrap_err();
2904 assert_eq!(error.code, "transport.timeout");
2905 closed_rx.recv().unwrap();
2906 let lost = transport
2907 .take_inbound()
2908 .into_iter()
2909 .filter(|inbound| matches!(inbound, Inbound::Disconnected(e) if e.code == "transport.timeout"))
2910 .count();
2911 assert_eq!(lost, 1, "the loss is reported once");
2912 server.join().unwrap();
2913 }
2914
2915 #[test]
2916 fn a_request_larger_than_a_chunk_rides_as_several_round_chunks() {
2917 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2918 let base = format!("http://{}", listener.local_addr().unwrap());
2919 let request: Vec<u8> = (0..3 * 64 * 1024 + 5).map(|i| (i % 251) as u8).collect();
2920 let expected = request.clone();
2921 let server = std::thread::spawn(move || {
2922 let (socket, _) = listener.accept().unwrap();
2923 let mut socket = tungstenite::accept(socket).unwrap();
2924 let mut body = Vec::new();
2925 let mut messages = 0;
2926 while body.len() < expected.len() {
2927 let message = socket.read().unwrap().into_data();
2928 assert_eq!(message[0], crate::REALTIME_TAG_ROUND);
2929 body.extend_from_slice(&message[1..]);
2930 messages += 1;
2931 }
2932 assert_eq!(body, expected);
2933 assert_eq!(messages, 4);
2934 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
2935 chunk.extend(minimal_response());
2936 socket
2937 .send(tungstenite::Message::Binary(chunk.into()))
2938 .unwrap();
2939 });
2940 let mut transport =
2941 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
2942 transport.realtime_connect_for_client("chunked").unwrap();
2943 let response = transport.realtime_sync(&request).unwrap();
2944 assert_eq!(response, minimal_response());
2945 transport.shutdown();
2946 server.join().unwrap();
2947 }
2948
2949 #[test]
2950 fn segment_http_failures_have_named_codes_and_safe_cause_details() {
2951 for signed in [false, true] {
2952 for body_failure in [false, true] {
2953 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2954 let base = format!("http://{}", listener.local_addr().unwrap());
2955 let server = std::thread::spawn(move || {
2956 let (mut socket, _) = listener.accept().unwrap();
2957 let mut reader = BufReader::new(socket.try_clone().unwrap());
2958 loop {
2959 let mut line = String::new();
2960 reader.read_line(&mut line).unwrap();
2961 if line == "\r\n" {
2962 break;
2963 }
2964 assert!(!line.is_empty());
2965 }
2966 socket.write_all(if body_failure {
2967 b"HTTP/1.1 200 OK\r\nContent-Length: 999\r\nConnection: close\r\n\r\nshort"
2968 } else {
2969 b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
2970 }).unwrap();
2971 });
2972 let mut transport =
2973 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
2974 let mut progress = |_| {};
2975 let error = if signed {
2976 transport.fetch_url(&format!("{base}/signed?signature=secret"), &mut progress)
2977 } else {
2978 transport.download_segment(
2979 &SegmentRequest {
2980 segment_id: "sha256:test".into(),
2981 table: "tasks".into(),
2982 requested_scopes_json: "{}".into(),
2983 },
2984 &mut progress,
2985 )
2986 }
2987 .unwrap_err();
2988 assert_eq!(error.code, "sync.transport_failed");
2989 assert_eq!(error.message, "segment transfer failed");
2990 let details = error.details.unwrap();
2991 assert!(details.get("path").is_none());
2992 assert!(details.get("causeMessage").is_none());
2993 assert_eq!(
2994 details["causeKind"],
2995 if body_failure { "body" } else { "status" }
2996 );
2997 assert_eq!(details["httpStatus"], if body_failure { 200 } else { 403 });
2998 server.join().unwrap();
2999 }
3000 }
3001 }
3002
3003 #[test]
3004 fn blob_status_classification_uses_the_actual_http_response() {
3005 for (status, expected) in [
3006 (404, "blob.not_found"),
3007 (401, "sync.auth_required"),
3008 (403, "blob.forbidden"),
3009 (500, "transport.failed"),
3010 ] {
3011 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3012 let base = format!("http://{}", listener.local_addr().unwrap());
3013 let server = std::thread::spawn(move || {
3014 let (mut socket, _) = listener.accept().unwrap();
3015 let mut reader = BufReader::new(socket.try_clone().unwrap());
3016 loop {
3017 let mut line = String::new();
3018 reader.read_line(&mut line).unwrap();
3019 if line == "\r\n" {
3020 break;
3021 }
3022 assert!(!line.is_empty());
3023 }
3024 let body = r#"{"code":"blob.not_found","message":"https://host/secret-path?secret-query#secret-fragment","retryable":false}"#;
3025 write!(socket, "HTTP/1.1 {status} Error\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();
3026 });
3027 let mut transport =
3028 HostTransport::new_from_config(&serde_json::json!({"baseUrl":base})).unwrap();
3029 let error = transport.blob_download("sha256:test").unwrap_err();
3030 assert_eq!(error.code, expected);
3031 assert_eq!(
3032 error.details,
3033 Some(serde_json::json!({
3034 "causeKind": "status",
3035 "httpStatus": status,
3036 "retryable": false
3037 }))
3038 );
3039 assert!(!error.message.contains("secret"));
3040 assert!(!error.message.contains("http"));
3041 server.join().unwrap();
3042 }
3043 }
3044
3045 #[test]
3046 fn refused_segment_connection_has_a_named_code_without_an_invented_status() {
3047 for signed in [false, true] {
3048 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3049 let base = format!("http://{}", listener.local_addr().unwrap());
3050 drop(listener);
3051 let mut transport =
3052 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3053 let mut progress = |_| {};
3054 let error = if signed {
3055 transport.fetch_url(&format!("{base}/signed?signature=secret"), &mut progress)
3056 } else {
3057 transport.download_segment(
3058 &SegmentRequest {
3059 segment_id: "sha256:test".into(),
3060 table: "tasks".into(),
3061 requested_scopes_json: "{}".into(),
3062 },
3063 &mut progress,
3064 )
3065 }
3066 .unwrap_err();
3067 assert_eq!(error.code, "sync.transport_failed");
3068 assert!(error.details.as_ref().unwrap().get("httpStatus").is_none());
3069 }
3070 }
3071
3072 #[test]
3073 fn segment_http_reports_intermediate_bytes_for_direct_and_signed_urls() {
3074 for signed in [false, true] {
3075 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3076 let base = format!("http://{}", listener.local_addr().unwrap());
3077 let server = std::thread::spawn(move || {
3078 let (mut socket, _) = listener.accept().unwrap();
3079 let mut reader = BufReader::new(socket.try_clone().unwrap());
3080 let mut headers = String::new();
3081 loop {
3082 let mut line = String::new();
3083 reader.read_line(&mut line).unwrap();
3084 if line == "\r\n" {
3085 break;
3086 }
3087 assert!(!line.is_empty());
3088 headers.push_str(&line.to_ascii_lowercase());
3089 }
3090 assert_eq!(headers.contains("authorization: bearer test"), !signed);
3091 assert_eq!(headers.contains("x-syncular-scopes:"), !signed);
3092 let bytes = vec![7; 128 * 1024 + 17];
3093 write!(
3094 socket,
3095 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
3096 bytes.len()
3097 )
3098 .unwrap();
3099 socket.write_all(&bytes).unwrap();
3100 });
3101 let mut transport = HostTransport::from_config(
3102 &serde_json::json!({"baseUrl": base, "headers": {"authorization": "Bearer test"}}),
3103 )
3104 .unwrap();
3105 let mut updates = Vec::new();
3106 let mut progress = |bytes| updates.push(bytes);
3107 let bytes = if signed {
3108 transport
3109 .fetch_url(&format!("{base}/signed"), &mut progress)
3110 .unwrap()
3111 } else {
3112 transport
3113 .download_segment(
3114 &SegmentRequest {
3115 segment_id: "sha256:test".into(),
3116 table: "tasks".into(),
3117 requested_scopes_json: "{}".into(),
3118 },
3119 &mut progress,
3120 )
3121 .unwrap()
3122 };
3123 assert_eq!(bytes.len(), 128 * 1024 + 17);
3124 assert!(updates.iter().any(|n| *n > 0 && *n < bytes.len() as u64));
3125 assert_eq!(updates.last(), Some(&(bytes.len() as u64)));
3126 assert!(updates.windows(2).all(|pair| pair[0] < pair[1]));
3127 server.join().unwrap();
3128 }
3129 }
3130
3131 #[test]
3132 fn request_over_cap_is_refused_before_network() {
3133 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3134 listener.set_nonblocking(true).unwrap();
3135 let base = format!("http://{}", listener.local_addr().unwrap());
3136 let mut transport =
3137 HostTransport::from_config(&serde_json::json!({"baseUrl": base, "maxRequestBytes": 4}))
3138 .unwrap();
3139 let error = transport
3140 .blob_upload("sha256:test", &[0u8; 8], None)
3141 .unwrap_err();
3142 assert_eq!(error.code, "transport.request_too_large");
3143 assert_eq!(
3144 error.details,
3145 Some(serde_json::json!({ "limit": 4, "size": 8 }))
3146 );
3147 assert!(
3148 matches!(listener.accept(), Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock),
3149 "the request cap is checked before any connection"
3150 );
3151 }
3152
3153 #[test]
3154 fn credential_bearing_redirects_are_refused_without_contacting_the_destination() {
3155 for credentials in ["headers", "userinfo", "password"] {
3156 for redirects in ["deny", "follow"] {
3157 let destination = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3158 destination.set_nonblocking(true).unwrap();
3159 let location = format!("http://{}/landed", destination.local_addr().unwrap());
3160 let origin = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3161 let host = origin.local_addr().unwrap();
3162 let (base, headers) = match credentials {
3163 "headers" => (
3164 format!("http://{host}"),
3165 serde_json::json!({"authorization": "Bearer secret"}),
3166 ),
3167 "userinfo" => (format!("http://user:secret@{host}"), serde_json::json!({})),
3168 _ => (format!("http://:secret@{host}"), serde_json::json!({})),
3169 };
3170 let server = std::thread::spawn(move || {
3171 let (mut socket, _) = origin.accept().unwrap();
3172 let mut reader = BufReader::new(socket.try_clone().unwrap());
3173 loop {
3174 let mut line = String::new();
3175 reader.read_line(&mut line).unwrap();
3176 if line == "\r\n" {
3177 break;
3178 }
3179 }
3180 let _ = write!(
3181 socket,
3182 "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
3183 );
3184 });
3185 let mut transport = HostTransport::from_config(&serde_json::json!({
3186 "baseUrl": base,
3187 "redirects": redirects,
3188 "headers": headers,
3189 }))
3190 .unwrap();
3191 let error = transport.blob_download("sha256:test").unwrap_err();
3192 assert_eq!(
3193 error.code, "transport.redirect",
3194 "{credentials} {redirects}"
3195 );
3196 let details = error.details.unwrap();
3197 assert_eq!(details["causeKind"], "redirect");
3198 assert_eq!(details["httpStatus"], 302);
3199 assert!(
3200 matches!(destination.accept(), Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock),
3201 "{credentials} {redirects}: the destination is never contacted"
3202 );
3203 server.join().unwrap();
3204 }
3205 }
3206 }
3207
3208 #[test]
3209 fn websocket_handshake_redirects_obey_the_redirect_policy() {
3210 for credential in ["headers", "userinfo", "query", "none"] {
3217 let redirects = "deny";
3218 let destination = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3219 let destination_addr = destination.local_addr().unwrap();
3220 let location = format!("ws://{destination_addr}/realtime");
3221 let landed = Arc::new(Mutex::new(Vec::new()));
3225 let landed_writer = Arc::clone(&landed);
3226 let destination_thread = std::thread::spawn(move || loop {
3227 let (mut socket, _) = destination.accept().unwrap();
3228 let mut reader = BufReader::new(socket.try_clone().unwrap());
3229 let mut head = String::new();
3230 loop {
3231 let mut line = String::new();
3232 reader.read_line(&mut line).unwrap();
3233 if line == "\r\n" {
3234 break;
3235 }
3236 head.push_str(&line);
3237 }
3238 let sentinel = head.starts_with("GET /sentinel");
3239 landed_writer.lock().unwrap().push(head);
3240 let _ = write!(
3241 socket,
3242 "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
3243 );
3244 if sentinel {
3245 break;
3246 }
3247 });
3248 let origin = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3249 let origin_addr = origin.local_addr().unwrap();
3250 let (headers, ws_url) = match credential {
3251 "headers" => (serde_json::json!({"authorization": "Bearer secret"}), None),
3252 "userinfo" => (
3253 serde_json::json!({}),
3254 Some(format!("ws://user:secret@{origin_addr}/realtime")),
3255 ),
3256 "query" => (
3257 serde_json::json!({}),
3258 Some(format!("ws://{origin_addr}/realtime?token=secret")),
3259 ),
3260 _ => (serde_json::json!({}), None),
3261 };
3262 let (head_tx, head_rx) = std::sync::mpsc::channel();
3263 let origin_thread = std::thread::spawn(move || {
3264 let (mut socket, _) = origin.accept().unwrap();
3265 let mut reader = BufReader::new(socket.try_clone().unwrap());
3266 let mut head = String::new();
3267 loop {
3268 let mut line = String::new();
3269 reader.read_line(&mut line).unwrap();
3270 if line == "\r\n" {
3271 break;
3272 }
3273 head.push_str(&line);
3274 }
3275 let _ = head_tx.send(head);
3276 let _ = write!(
3277 socket,
3278 "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
3279 );
3280 });
3281 let mut config = serde_json::json!({
3282 "baseUrl": format!("http://{origin_addr}"),
3283 "headers": headers,
3284 "redirects": redirects,
3285 });
3286 if let Some(ws_url) = &ws_url {
3287 config["wsUrl"] = serde_json::json!(ws_url);
3288 }
3289 let mut transport = HostTransport::from_config(&config).unwrap();
3290 let result = transport.realtime_connect();
3291 let mut sentinel = std::net::TcpStream::connect(destination_addr).unwrap();
3294 let _ = write!(sentinel, "GET /sentinel HTTP/1.1\r\nHost: sentinel\r\n\r\n");
3295 destination_thread.join().unwrap();
3296 let landed = landed.lock().unwrap().clone();
3297 origin_thread.join().unwrap();
3298 let error = result.unwrap_err();
3299 assert_eq!(
3300 error.code, "transport.redirect",
3301 "{credential}: a refused handshake redirect surfaces the redirect code"
3302 );
3303 let details = error.details.unwrap();
3304 assert_eq!(details["causeKind"], "redirect");
3305 assert_eq!(details["httpStatus"], 302);
3306 assert_eq!(
3307 landed.len(),
3308 1,
3309 "{credential}: the redirect target is never contacted: {landed:?}"
3310 );
3311 assert!(landed[0].starts_with("GET /sentinel"), "{landed:?}");
3312 let head = head_rx.recv().unwrap();
3313 if credential == "userinfo" {
3314 assert!(
3315 !head.contains("secret"),
3316 "the handshake request line drops URL userinfo"
3317 );
3318 }
3319 if credential == "query" {
3320 assert!(
3321 head.contains("token=secret"),
3322 "the handshake request line carries the realtime query"
3323 );
3324 }
3325 }
3326 }
3327
3328 const GZIP_BOMB: [u8; 548] = [
3331 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xed, 0xc9, 0xb1, 0x11, 0x80,
3332 0x20, 0x10, 0x00, 0xb0, 0x59, 0x1c, 0x80, 0xa1, 0xfe, 0x90, 0xb3, 0x41, 0xf0, 0x7c, 0x2d,
3333 0xd8, 0xde, 0x09, 0xdc, 0x20, 0x69, 0x93, 0x6b, 0xd4, 0xb7, 0xc7, 0x5d, 0xb2, 0x1d, 0x67,
3334 0x1b, 0x4f, 0xb9, 0x62, 0xf5, 0x19, 0xfb, 0x96, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3335 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3336 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3337 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3338 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3339 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3340 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3341 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3342 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3343 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3344 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3345 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3346 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3347 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3348 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3349 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3350 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3351 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3352 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3353 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3354 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3355 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3356 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3357 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3358 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3359 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3360 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3361 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3362 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3363 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3364 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3365 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3366 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0xf8, 0x8b, 0x0f,
3367 0xdb, 0x82, 0xf4, 0xa9, 0x40, 0x0d, 0x03, 0x00,
3368 ];
3369
3370 #[test]
3371 fn decoded_response_cap_counts_decompressed_bytes() {
3372 for cap in [200000u64, 199999] {
3373 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3374 let base = format!("http://{}", listener.local_addr().unwrap());
3375 let server = std::thread::spawn(move || {
3376 let (mut socket, _) = listener.accept().unwrap();
3377 let mut reader = BufReader::new(socket.try_clone().unwrap());
3378 loop {
3379 let mut line = String::new();
3380 reader.read_line(&mut line).unwrap();
3381 if line == "\r\n" {
3382 break;
3383 }
3384 }
3385 let _ = write!(
3386 socket,
3387 "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
3388 GZIP_BOMB.len()
3389 );
3390 let _ = socket.write_all(&GZIP_BOMB);
3391 });
3392 let mut transport = HostTransport::from_config(
3393 &serde_json::json!({"baseUrl": base, "maxResponseBytes": cap}),
3394 )
3395 .unwrap();
3396 let result = transport.fetch_blob_url(&format!("{base}/signed"));
3397 if cap == 200000 {
3398 let body = result.expect("a cap equal to the decoded size admits the body");
3399 assert_eq!(body.len(), 200000);
3400 } else {
3401 let error = result.unwrap_err();
3402 assert_eq!(error.code, "transport.response_too_large");
3403 assert_eq!(error.details, Some(serde_json::json!({ "limit": cap })));
3404 }
3405 server.join().unwrap();
3406 }
3407 }
3408
3409 #[test]
3410 fn a_json_http_error_body_reaches_the_caller() {
3411 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3412 let base = format!("http://{}", listener.local_addr().unwrap());
3413 let body = r#"{"code":"sync.invalid_client_id","category":"invalid-request","retryable":false,"recommendedAction":"resetClientId","message":"clientId is bound to a different actor in this partition (§1.5)","details":{"actorId":"actor-b"}}"#;
3414 let server = std::thread::spawn(move || {
3415 let (mut socket, _) = listener.accept().unwrap();
3416 let mut reader = BufReader::new(socket.try_clone().unwrap());
3417 loop {
3418 let mut line = String::new();
3419 reader.read_line(&mut line).unwrap();
3420 if line == "\r\n" {
3421 break;
3422 }
3423 }
3424 let _ = write!(
3425 socket,
3426 "HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
3427 body.len(),
3428 body
3429 );
3430 });
3431 let mut transport =
3432 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3433 let error = transport.sync(&[]).unwrap_err();
3434 assert_eq!(error.code, "sync.invalid_client_id");
3435 assert_eq!(
3436 error.message,
3437 "clientId is bound to a different actor in this partition (§1.5)"
3438 );
3439 let details = error.details.unwrap();
3440 assert_eq!(details["recommendedAction"], "resetClientId");
3441 assert_eq!(details["category"], "invalid-request");
3442 assert_eq!(details["retryable"], false);
3443 assert_eq!(details["httpStatus"], 400);
3444 assert_eq!(details["actorId"], "actor-b");
3445 server.join().unwrap();
3446 }
3447}