1use std::sync::atomic::{AtomicU64, Ordering};
24use std::sync::{mpsc, Arc, Mutex};
25use std::time::{Duration, SystemTime};
26
27use crate::{
28 BlobDownload, BlobUploadGrant, SegmentFetch, SegmentRequest, Transport, TransportError,
29};
30
31#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
33pub enum RedirectPolicy {
34 #[default]
39 Deny,
40 Follow,
50}
51
52#[derive(Debug, Clone, Default)]
55pub struct HostTransportPolicy {
56 pub request_timeout: Option<Duration>,
58 pub round_deadline: Option<Duration>,
63 pub max_request_bytes: Option<u64>,
66 pub max_response_bytes: Option<u64>,
69 pub redirects: RedirectPolicy,
70 pub realtime_heartbeat: Option<Duration>,
74}
75
76impl HostTransportPolicy {
77 pub fn validate(&self) -> Result<(), String> {
81 let now = std::time::Instant::now();
82 if let Some(timeout) = self.request_timeout {
83 if timeout.is_zero() || now.checked_add(timeout).is_none() {
84 return Err(
85 "sync.invalid_request: transport config requestTimeoutMs is out of range"
86 .to_owned(),
87 );
88 }
89 }
90 if let Some(deadline) = self.round_deadline {
91 if deadline.is_zero() || now.checked_add(deadline).is_none() {
92 return Err(
93 "sync.invalid_request: transport config roundDeadlineMs is out of range"
94 .to_owned(),
95 );
96 }
97 }
98 if let Some(interval) = self.realtime_heartbeat {
99 if interval.is_zero()
100 || interval
101 .checked_mul(2)
102 .and_then(|silence| now.checked_add(silence))
103 .is_none()
104 {
105 return Err(
106 "sync.invalid_request: transport config realtimeHeartbeatMs is out of range"
107 .to_owned(),
108 );
109 }
110 }
111 if self.max_request_bytes == Some(0) {
112 return Err(
113 "sync.invalid_request: transport config maxRequestBytes must be a positive integer"
114 .to_owned(),
115 );
116 }
117 if self.max_response_bytes == Some(0) {
118 return Err("sync.invalid_request: transport config maxResponseBytes must be a positive integer".to_owned());
119 }
120 Ok(())
121 }
122
123 pub fn from_config(config: &serde_json::Value) -> Result<Self, String> {
126 let bound = |key: &str, message: &'static str| -> Result<Option<u64>, String> {
127 match config.get(key) {
128 None | Some(serde_json::Value::Null) => Ok(None),
129 Some(value) => match value.as_u64() {
130 Some(number) if number >= 1 => Ok(Some(number)),
131 _ => Err(message.to_owned()),
132 },
133 }
134 };
135 let redirects = match config.get("redirects") {
136 None | Some(serde_json::Value::Null) => RedirectPolicy::Deny,
137 Some(serde_json::Value::String(value)) if value == "deny" => RedirectPolicy::Deny,
138 Some(serde_json::Value::String(value)) if value == "follow" => RedirectPolicy::Follow,
139 Some(_) => return Err(
140 "sync.invalid_request: transport config redirects must be \"deny\" or \"follow\""
141 .to_owned(),
142 ),
143 };
144 let policy = Self {
145 request_timeout: bound(
146 "requestTimeoutMs",
147 "sync.invalid_request: transport config requestTimeoutMs must be a positive integer",
148 )?
149 .map(Duration::from_millis),
150 round_deadline: bound(
151 "roundDeadlineMs",
152 "sync.invalid_request: transport config roundDeadlineMs must be a positive integer",
153 )?
154 .map(Duration::from_millis),
155 max_request_bytes: bound(
156 "maxRequestBytes",
157 "sync.invalid_request: transport config maxRequestBytes must be a positive integer",
158 )?,
159 max_response_bytes: bound(
160 "maxResponseBytes",
161 "sync.invalid_request: transport config maxResponseBytes must be a positive integer",
162 )?,
163 redirects,
164 realtime_heartbeat: bound(
165 "realtimeHeartbeatMs",
166 "sync.invalid_request: transport config realtimeHeartbeatMs must be a positive integer",
167 )?
168 .map(Duration::from_millis),
169 };
170 policy.validate()?;
171 Ok(policy)
172 }
173}
174
175pub enum Inbound {
177 Text(String),
178 Binary(Vec<u8>),
179 Disconnected(TransportError),
182}
183
184#[derive(Debug, Clone, Copy, PartialEq, Eq)]
187pub struct RealtimeReceipt {
188 pub received_at: SystemTime,
189 pub bytes: usize,
190 pub kind: RealtimeReceiptKind,
191}
192
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
194pub enum RealtimeReceiptKind {
195 Text,
196 Delta,
197 Round,
198 OtherBinary,
199 Ping,
200 Pong,
201 Close,
202}
203
204pub const INBOUND_BUFFER_LIMIT_BYTES: usize = 16 * 1024 * 1024;
208
209#[derive(Default)]
211struct InboundFrames {
212 list: Vec<Inbound>,
213 bytes: usize,
214}
215
216pub struct InboundBuffer {
220 frames: Mutex<InboundFrames>,
221 notify: Option<Arc<dyn Fn() + Send + Sync>>,
222 receipts: Mutex<Option<mpsc::SyncSender<RealtimeReceipt>>>,
223 dropped_receipts: AtomicU64,
224}
225
226impl Default for InboundBuffer {
227 fn default() -> Self {
228 Self {
229 frames: Mutex::new(InboundFrames::default()),
230 notify: None,
231 receipts: Mutex::new(None),
232 dropped_receipts: AtomicU64::new(0),
233 }
234 }
235}
236
237impl InboundBuffer {
238 pub fn with_notify(notify: Arc<dyn Fn() + Send + Sync>) -> Self {
239 Self {
240 frames: Mutex::new(InboundFrames::default()),
241 notify: Some(notify),
242 receipts: Mutex::new(None),
243 dropped_receipts: AtomicU64::new(0),
244 }
245 }
246
247 pub fn push(&self, frame: Inbound) {
248 {
249 let mut frames = self.frames.lock().expect("inbound lock");
250 frames.bytes += match &frame {
251 Inbound::Text(text) => text.len(),
252 Inbound::Binary(bytes) => bytes.len(),
253 Inbound::Disconnected(_) => 0,
254 };
255 frames.list.push(frame);
256 }
257 if let Some(notify) = &self.notify {
258 notify();
259 }
260 }
261
262 fn take(&self) -> Vec<Inbound> {
263 std::mem::take(&mut *self.frames.lock().expect("inbound lock")).list
264 }
265
266 #[cfg(feature = "native-transport")]
268 fn overflowed(&self) -> bool {
269 self.frames.lock().expect("inbound lock").bytes > INBOUND_BUFFER_LIMIT_BYTES
270 }
271
272 #[cfg(feature = "native-transport")]
273 fn observe(&self, kind: RealtimeReceiptKind, bytes: usize) {
274 if let Some(sender) = self.receipts.lock().expect("receipt lock").as_ref() {
275 let receipt = RealtimeReceipt {
276 received_at: SystemTime::now(),
277 bytes,
278 kind,
279 };
280 if sender.try_send(receipt).is_err() {
281 self.dropped_receipts.fetch_add(1, Ordering::Relaxed);
282 }
283 }
284 }
285}
286
287#[allow(clippy::large_enum_variant)]
290pub enum HostTransport {
291 Null {
293 signed_urls: bool,
294 inbound: Arc<InboundBuffer>,
295 },
296 #[cfg(feature = "native-transport")]
297 Native(native::NativeTransport),
298 #[cfg(feature = "local-authority")]
300 Local(Box<LocalAuthorityTransport<HostTransport>>),
301}
302
303impl HostTransport {
304 pub fn from_config(config: &serde_json::Value) -> Result<Self, String> {
307 Self::new_from_config(config)
308 }
309
310 pub fn new_from_config(config: &serde_json::Value) -> Result<Self, String> {
313 Self::from_config_with_notify(config, None)
314 }
315
316 pub fn from_config_with_notify(
320 config: &serde_json::Value,
321 notify: Option<Arc<dyn Fn() + Send + Sync>>,
322 ) -> Result<Self, String> {
323 let policy = HostTransportPolicy::from_config(config)?;
324 Self::from_config_with_policy(config, policy, notify)
325 }
326
327 pub fn from_config_with_policy(
331 config: &serde_json::Value,
332 policy: HostTransportPolicy,
333 notify: Option<Arc<dyn Fn() + Send + Sync>>,
334 ) -> Result<Self, String> {
335 let local_authority_path = match config.get("localAuthorityPath") {
336 None | Some(serde_json::Value::Null) => None,
337 Some(serde_json::Value::String(path)) if !path.is_empty() => Some(path.clone()),
338 Some(_) => return Err("localAuthorityPath must be a non-empty string".to_owned()),
339 };
340 #[cfg(not(feature = "local-authority"))]
341 if local_authority_path.is_some() {
342 return Err(format!(
343 "{}: this build has no local authority (rebuild with --features local-authority)",
344 crate::LOCAL_AUTHORITY_UNCONFIGURED_CODE
345 ));
346 }
347 let remote = match config.get("baseUrl").and_then(|v| v.as_str()) {
348 #[cfg(feature = "native-transport")]
349 Some(base_url) => HostTransport::Native(native::NativeTransport::with_policy(
350 base_url, config, policy, notify,
351 )?),
352 _ => {
353 #[cfg(not(feature = "native-transport"))]
354 {
355 policy.validate()?;
356 if config.get("baseUrl").is_some() {
357 return Err(
358 "this build has no native transport (rebuild with --features native-transport)"
359 .to_owned(),
360 );
361 }
362 }
363 HostTransport::Null {
364 signed_urls: false,
365 inbound: Arc::new(match notify {
366 Some(notify) => InboundBuffer::with_notify(notify),
367 None => InboundBuffer::default(),
368 }),
369 }
370 }
371 };
372 #[cfg(feature = "local-authority")]
373 if let Some(path) = local_authority_path {
374 let remote_configured = !matches!(remote, HostTransport::Null { .. });
375 return Ok(HostTransport::Local(Box::new(
376 LocalAuthorityTransport::new(path, remote, remote_configured),
377 )));
378 }
379 Ok(remote)
380 }
381
382 pub fn fork_round(&self) -> Self {
385 match self {
386 Self::Null {
387 signed_urls,
388 inbound,
389 } => Self::Null {
390 signed_urls: *signed_urls,
391 inbound: inbound.clone(),
392 },
393 #[cfg(feature = "native-transport")]
394 Self::Native(t) => Self::Native(t.fork_round()),
395 #[cfg(feature = "local-authority")]
396 Self::Local(t) => Self::Local(Box::new(t.fork_round())),
397 }
398 }
399
400 pub fn set_signed_urls(&mut self, value: bool) {
401 match self {
402 HostTransport::Null { signed_urls, .. } => *signed_urls = value,
403 #[cfg(feature = "native-transport")]
404 HostTransport::Native(t) => t.signed_urls = value,
405 #[cfg(feature = "local-authority")]
406 HostTransport::Local(t) => t.remote.set_signed_urls(value),
407 }
408 }
409
410 pub fn set_headers(&mut self, headers: Vec<(String, String)>) {
413 match self {
414 HostTransport::Null { .. } => drop(headers),
415 #[cfg(feature = "native-transport")]
416 HostTransport::Native(t) => t.set_headers(headers),
417 #[cfg(feature = "local-authority")]
418 HostTransport::Local(t) => t.remote.set_headers(headers),
419 }
420 }
421
422 pub fn set_policy(&mut self, policy: HostTransportPolicy) -> Result<(), String> {
427 policy.validate()?;
428 match self {
429 HostTransport::Null { .. } => {}
430 #[cfg(feature = "native-transport")]
431 HostTransport::Native(t) => t.set_policy(policy)?,
432 #[cfg(feature = "local-authority")]
433 HostTransport::Local(t) => t.remote.set_policy(policy)?,
434 }
435 Ok(())
436 }
437
438 pub fn take_inbound(&mut self) -> Vec<Inbound> {
440 match self {
441 HostTransport::Null { inbound, .. } => inbound.take(),
442 #[cfg(feature = "native-transport")]
443 HostTransport::Native(t) => t.inbound.take(),
444 #[cfg(feature = "local-authority")]
445 HostTransport::Local(t) => t.remote.take_inbound(),
446 }
447 }
448
449 pub fn set_realtime_receipt_observer(
454 &mut self,
455 sender: Option<mpsc::SyncSender<RealtimeReceipt>>,
456 ) {
457 let inbound = match self {
458 HostTransport::Null { inbound, .. } => inbound,
459 #[cfg(feature = "native-transport")]
460 HostTransport::Native(t) => &t.inbound,
461 #[cfg(feature = "local-authority")]
462 HostTransport::Local(t) => {
463 t.remote.set_realtime_receipt_observer(sender);
464 return;
465 }
466 };
467 let mut current = inbound.receipts.lock().expect("receipt lock");
468 *current = sender;
469 inbound.dropped_receipts.store(0, Ordering::Relaxed);
470 }
471
472 pub fn take_dropped_realtime_receipts(&mut self) -> u64 {
474 let inbound = match self {
475 HostTransport::Null { inbound, .. } => inbound,
476 #[cfg(feature = "native-transport")]
477 HostTransport::Native(t) => &t.inbound,
478 #[cfg(feature = "local-authority")]
479 HostTransport::Local(t) => return t.remote.take_dropped_realtime_receipts(),
480 };
481 inbound.dropped_receipts.swap(0, Ordering::Relaxed)
482 }
483
484 pub fn shutdown(&mut self) {
487 match self {
488 HostTransport::Null { .. } => {}
489 #[cfg(feature = "native-transport")]
490 HostTransport::Native(t) => t.shutdown(),
491 #[cfg(feature = "local-authority")]
492 HostTransport::Local(t) => t.shutdown(),
493 }
494 }
495}
496
497fn unavailable(op: &str) -> TransportError {
498 TransportError::new(
499 "transport.unavailable",
500 format!("{op} needs the native transport (build with --features native-transport)"),
501 )
502}
503
504#[cfg_attr(not(feature = "native-transport"), allow(unused_variables))]
507impl Transport for HostTransport {
508 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
509 match self {
510 HostTransport::Null { .. } => Err(unavailable("sync")),
511 #[cfg(feature = "native-transport")]
512 HostTransport::Native(t) => t.sync(request),
513 #[cfg(feature = "local-authority")]
514 HostTransport::Local(t) => t.sync(request),
515 }
516 }
517
518 fn remote_operation(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
519 match self {
520 HostTransport::Null { .. } => Err(unavailable("remoteOperation")),
521 #[cfg(feature = "native-transport")]
522 HostTransport::Native(t) => t.remote_operation(request),
523 #[cfg(feature = "local-authority")]
524 HostTransport::Local(t) => t.remote_operation(request),
525 }
526 }
527
528 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
529 match self {
530 HostTransport::Null { .. } => Err(unavailable("realtimeSync")),
531 #[cfg(feature = "native-transport")]
532 HostTransport::Native(t) => t.realtime_sync(request),
533 #[cfg(feature = "local-authority")]
534 HostTransport::Local(t) => t.realtime_sync(request),
535 }
536 }
537
538 fn download_segment(
539 &mut self,
540 request: &SegmentRequest,
541 on_progress: &mut dyn FnMut(u64),
542 ) -> Result<Vec<u8>, TransportError> {
543 match self {
544 HostTransport::Null { .. } => Err(unavailable("downloadSegment")),
545 #[cfg(feature = "native-transport")]
546 HostTransport::Native(t) => t.download_segment(request, on_progress),
547 #[cfg(feature = "local-authority")]
548 HostTransport::Local(t) => t.download_segment(request, on_progress),
549 }
550 }
551
552 fn supports_url_fetch(&self) -> bool {
553 match self {
554 HostTransport::Null { signed_urls, .. } => *signed_urls,
555 #[cfg(feature = "native-transport")]
556 HostTransport::Native(t) => t.signed_urls,
557 #[cfg(feature = "local-authority")]
558 HostTransport::Local(t) => t.supports_url_fetch(),
559 }
560 }
561
562 fn download_segments(
563 &mut self,
564 fetches: &[SegmentFetch],
565 on_progress: &mut (dyn FnMut(usize, u64) + Send),
566 ) -> Vec<Result<Vec<u8>, TransportError>> {
567 match self {
568 HostTransport::Null { .. } => fetches
569 .first()
570 .map(|_| Err(unavailable("downloadSegments")))
571 .into_iter()
572 .collect(),
573 #[cfg(feature = "native-transport")]
574 HostTransport::Native(t) => t.download_segments(fetches, on_progress),
575 #[cfg(feature = "local-authority")]
576 HostTransport::Local(t) => t.download_segments(fetches, on_progress),
577 }
578 }
579
580 fn fetch_url(
581 &mut self,
582 url: &str,
583 byte_length: u64,
584 on_progress: &mut dyn FnMut(u64),
585 ) -> Result<Vec<u8>, TransportError> {
586 match self {
587 HostTransport::Null { .. } => Err(unavailable("fetchUrl")),
588 #[cfg(feature = "native-transport")]
589 HostTransport::Native(t) => t.fetch_url(url, byte_length, on_progress),
590 #[cfg(feature = "local-authority")]
591 HostTransport::Local(t) => t.fetch_url(url, byte_length, on_progress),
592 }
593 }
594
595 fn blob_upload(
596 &mut self,
597 blob_id: &str,
598 bytes: &[u8],
599 media_type: Option<&str>,
600 ) -> Result<(), TransportError> {
601 match self {
602 HostTransport::Null { .. } => Err(unavailable("blobUpload")),
603 #[cfg(feature = "native-transport")]
604 HostTransport::Native(t) => t.blob_upload(blob_id, bytes, media_type),
605 #[cfg(feature = "local-authority")]
606 HostTransport::Local(t) => t.blob_upload(blob_id, bytes, media_type),
607 }
608 }
609
610 fn blob_download(
611 &mut self,
612 blob_id: &str,
613 byte_length: Option<u64>,
614 ) -> Result<BlobDownload, TransportError> {
615 match self {
616 HostTransport::Null { .. } => Err(unavailable("blobDownload")),
617 #[cfg(feature = "native-transport")]
618 HostTransport::Native(t) => t.blob_download(blob_id, byte_length),
619 #[cfg(feature = "local-authority")]
620 HostTransport::Local(t) => t.blob_download(blob_id, byte_length),
621 }
622 }
623
624 fn fetch_blob_url(
625 &mut self,
626 url: &str,
627 byte_length: Option<u64>,
628 ) -> Result<Vec<u8>, TransportError> {
629 match self {
630 HostTransport::Null { .. } => Err(unavailable("fetchBlobUrl")),
631 #[cfg(feature = "native-transport")]
632 HostTransport::Native(t) => t.fetch_blob_url(url, byte_length),
633 #[cfg(feature = "local-authority")]
634 HostTransport::Local(t) => t.fetch_blob_url(url, byte_length),
635 }
636 }
637
638 fn blob_upload_grant(
639 &mut self,
640 blob_id: &str,
641 byte_length: u64,
642 media_type: Option<&str>,
643 ) -> Result<BlobUploadGrant, TransportError> {
644 match self {
645 HostTransport::Null { .. } => Ok(BlobUploadGrant::None),
648 #[cfg(feature = "native-transport")]
649 HostTransport::Native(t) => t.blob_upload_grant(blob_id, byte_length, media_type),
650 #[cfg(feature = "local-authority")]
651 HostTransport::Local(t) => t.blob_upload_grant(blob_id, byte_length, media_type),
652 }
653 }
654
655 fn blob_put_url(
656 &mut self,
657 blob_id: &str,
658 url: &str,
659 bytes: &[u8],
660 media_type: Option<&str>,
661 ) -> Result<(), TransportError> {
662 match self {
663 HostTransport::Null { .. } => Err(unavailable("blobPutUrl")),
664 #[cfg(feature = "native-transport")]
665 HostTransport::Native(t) => t.blob_put_url(blob_id, url, bytes, media_type),
666 #[cfg(feature = "local-authority")]
667 HostTransport::Local(t) => t.blob_put_url(blob_id, url, bytes, media_type),
668 }
669 }
670
671 fn realtime_connect(&mut self) -> Result<(), TransportError> {
672 match self {
673 HostTransport::Null { .. } => Err(unavailable("realtimeConnect")),
674 #[cfg(feature = "native-transport")]
675 HostTransport::Native(t) => {
676 t.set_realtime_client_id(None);
677 t.realtime_connect()
678 }
679 #[cfg(feature = "local-authority")]
680 HostTransport::Local(t) => t.realtime_connect(),
681 }
682 }
683
684 fn realtime_connect_for_client(&mut self, client_id: &str) -> Result<(), TransportError> {
685 match self {
686 HostTransport::Null { .. } => Err(unavailable("realtimeConnect")),
687 #[cfg(feature = "native-transport")]
688 HostTransport::Native(t) => {
689 t.set_realtime_client_id(Some(client_id));
690 t.realtime_connect()
691 }
692 #[cfg(feature = "local-authority")]
693 HostTransport::Local(t) => t.realtime_connect_for_client(client_id),
694 }
695 }
696
697 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
698 match self {
699 HostTransport::Null { .. } => Err(unavailable("realtimeSend")),
700 #[cfg(feature = "native-transport")]
701 HostTransport::Native(t) => t.realtime_send(text),
702 #[cfg(feature = "local-authority")]
703 HostTransport::Local(t) => t.realtime_send(text),
704 }
705 }
706
707 fn realtime_close(&mut self) -> Result<(), TransportError> {
708 match self {
709 HostTransport::Null { .. } => Ok(()),
710 #[cfg(feature = "native-transport")]
711 HostTransport::Native(t) => t.realtime_close(),
712 #[cfg(feature = "local-authority")]
713 HostTransport::Local(t) => t.realtime_close(),
714 }
715 }
716
717 fn round_deadline(&self) -> Option<std::time::Duration> {
718 match self {
719 HostTransport::Null { .. } => None,
720 #[cfg(feature = "native-transport")]
721 HostTransport::Native(t) => t.round_deadline(),
722 #[cfg(feature = "local-authority")]
723 HostTransport::Local(t) => t.round_deadline(),
724 }
725 }
726
727 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
728 match self {
729 HostTransport::Null { .. } => {}
730 #[cfg(feature = "native-transport")]
731 HostTransport::Native(t) => t.set_round_deadline(deadline),
732 #[cfg(feature = "local-authority")]
733 HostTransport::Local(t) => t.set_round_deadline(deadline),
734 }
735 }
736
737 #[cfg(feature = "local-authority")]
738 fn open_local_authority(
739 &mut self,
740 schema: &serde_json::Value,
741 replica_path: Option<&str>,
742 ) -> Result<(), TransportError> {
743 match self {
744 HostTransport::Local(t) => t.open_local_authority(schema, replica_path),
745 _ => Ok(()),
746 }
747 }
748
749 #[cfg(feature = "local-authority")]
750 fn local_authority_state(&self) -> Option<&'static str> {
751 match self {
752 HostTransport::Local(t) => t.local_authority_state(),
753 _ => None,
754 }
755 }
756
757 #[cfg(feature = "local-authority")]
758 fn attach_local_authority(&mut self) -> Result<(), TransportError> {
759 match self {
760 HostTransport::Local(t) => t.attach_local_authority(),
761 _ => Err(local::unconfigured("this transport has no local authority")),
762 }
763 }
764
765 #[cfg(feature = "local-authority")]
766 fn abandon_local_authority_attach(&mut self) -> Result<(), TransportError> {
767 match self {
768 HostTransport::Local(t) => t.abandon_local_authority_attach(),
769 _ => Err(local::unconfigured("this transport has no local authority")),
770 }
771 }
772}
773
774#[cfg(feature = "local-authority")]
775pub use local::LocalAuthorityTransport;
776
777#[cfg(feature = "local-authority")]
778mod local {
779 use std::sync::{Arc, Mutex};
787
788 use syncular_authority::{AuthorityError, AuthorityState, LocalAuthority};
789
790 use super::HostTransport;
791 use crate::{
792 BlobDownload, BlobUploadGrant, SegmentFetch, SegmentRequest, Transport, TransportError,
793 LOCAL_AUTHORITY_UNCONFIGURED_CODE,
794 };
795
796 pub(super) fn unconfigured(message: &'static str) -> TransportError {
797 TransportError::new(LOCAL_AUTHORITY_UNCONFIGURED_CODE, message)
798 }
799
800 fn transport_error(error: AuthorityError) -> TransportError {
801 TransportError {
802 code: error.code,
803 message: error.message,
804 details: error.details,
805 }
806 }
807
808 fn authority_error(error: TransportError) -> AuthorityError {
809 AuthorityError {
810 code: error.code,
811 message: error.message,
812 details: error.details,
813 }
814 }
815
816 fn refused(message: &'static str) -> TransportError {
817 TransportError::new("sync.invalid_request", message)
818 }
819
820 pub struct LocalAuthorityTransport<R> {
822 path: String,
823 authority: Arc<Mutex<Option<LocalAuthority>>>,
826 pub(super) remote: R,
827 remote_configured: bool,
830 }
831
832 impl<R> LocalAuthorityTransport<R> {
833 pub fn new(path: impl Into<String>, remote: R, remote_configured: bool) -> Self {
837 Self {
838 path: path.into(),
839 authority: Arc::new(Mutex::new(None)),
840 remote,
841 remote_configured,
842 }
843 }
844
845 fn local_active(&self) -> bool {
846 self.authority
847 .lock()
848 .expect("local authority lock")
849 .as_ref()
850 .is_none_or(|authority| authority.state() != AuthorityState::Attached)
851 }
852 }
853
854 impl LocalAuthorityTransport<HostTransport> {
855 pub(super) fn fork_round(&self) -> Self {
856 Self {
857 path: self.path.clone(),
858 authority: self.authority.clone(),
859 remote: self.remote.fork_round(),
860 remote_configured: self.remote_configured,
861 }
862 }
863
864 pub(super) fn shutdown(&mut self) {
865 self.remote.shutdown();
866 *self.authority.lock().expect("local authority lock") = None;
867 }
868 }
869
870 impl<R: Transport> Transport for LocalAuthorityTransport<R> {
871 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
872 let mut authority = self.authority.lock().expect("local authority lock");
873 match authority.as_mut() {
874 None => Err(unconfigured(
875 "the local authority opens with create; create a client first",
876 )),
877 Some(local) if local.state() != AuthorityState::Attached => {
878 local.sync(request).map_err(transport_error)
879 }
880 Some(_) => {
881 drop(authority);
882 self.remote.sync(request)
883 }
884 }
885 }
886
887 fn remote_operation(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
888 self.remote.remote_operation(request)
889 }
890
891 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
892 if self.local_active() {
893 return Err(refused("the local authority has no realtime channel"));
894 }
895 self.remote.realtime_sync(request)
896 }
897
898 fn download_segment(
899 &mut self,
900 request: &SegmentRequest,
901 on_progress: &mut dyn FnMut(u64),
902 ) -> Result<Vec<u8>, TransportError> {
903 if self.local_active() {
904 return Err(refused("the local authority has no segment endpoint"));
905 }
906 self.remote.download_segment(request, on_progress)
907 }
908
909 fn download_segments(
910 &mut self,
911 fetches: &[SegmentFetch],
912 on_progress: &mut (dyn FnMut(usize, u64) + Send),
913 ) -> Vec<Result<Vec<u8>, TransportError>> {
914 if self.local_active() {
915 return fetches
916 .first()
917 .map(|_| Err(refused("the local authority has no segment endpoint")))
918 .into_iter()
919 .collect();
920 }
921 self.remote.download_segments(fetches, on_progress)
922 }
923
924 fn supports_url_fetch(&self) -> bool {
925 self.remote.supports_url_fetch()
926 }
927
928 fn fetch_url(
929 &mut self,
930 url: &str,
931 byte_length: u64,
932 on_progress: &mut dyn FnMut(u64),
933 ) -> Result<Vec<u8>, TransportError> {
934 if self.local_active() {
935 return Err(refused("the local authority has no segment endpoint"));
936 }
937 self.remote.fetch_url(url, byte_length, on_progress)
938 }
939
940 fn blob_upload(
941 &mut self,
942 blob_id: &str,
943 bytes: &[u8],
944 media_type: Option<&str>,
945 ) -> Result<(), TransportError> {
946 self.remote.blob_upload(blob_id, bytes, media_type)
947 }
948
949 fn blob_download(
950 &mut self,
951 blob_id: &str,
952 byte_length: Option<u64>,
953 ) -> Result<BlobDownload, TransportError> {
954 self.remote.blob_download(blob_id, byte_length)
955 }
956
957 fn fetch_blob_url(
958 &mut self,
959 url: &str,
960 byte_length: Option<u64>,
961 ) -> Result<Vec<u8>, TransportError> {
962 self.remote.fetch_blob_url(url, byte_length)
963 }
964
965 fn blob_upload_grant(
966 &mut self,
967 blob_id: &str,
968 byte_length: u64,
969 media_type: Option<&str>,
970 ) -> Result<BlobUploadGrant, TransportError> {
971 self.remote
972 .blob_upload_grant(blob_id, byte_length, media_type)
973 }
974
975 fn blob_put_url(
976 &mut self,
977 blob_id: &str,
978 url: &str,
979 bytes: &[u8],
980 media_type: Option<&str>,
981 ) -> Result<(), TransportError> {
982 self.remote.blob_put_url(blob_id, url, bytes, media_type)
983 }
984
985 fn realtime_connect(&mut self) -> Result<(), TransportError> {
986 if self.local_active() {
987 return Err(refused("the local authority has no realtime channel"));
988 }
989 self.remote.realtime_connect()
990 }
991
992 fn realtime_connect_for_client(&mut self, client_id: &str) -> Result<(), TransportError> {
993 if self.local_active() {
994 return Err(refused("the local authority has no realtime channel"));
995 }
996 self.remote.realtime_connect_for_client(client_id)
997 }
998
999 fn round_deadline(&self) -> Option<std::time::Duration> {
1000 self.remote.round_deadline()
1001 }
1002
1003 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
1004 self.remote.set_round_deadline(deadline);
1005 }
1006
1007 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
1008 self.remote.realtime_send(text)
1009 }
1010
1011 fn realtime_close(&mut self) -> Result<(), TransportError> {
1012 self.remote.realtime_close()
1013 }
1014
1015 fn open_local_authority(
1016 &mut self,
1017 schema: &serde_json::Value,
1018 replica_path: Option<&str>,
1019 ) -> Result<(), TransportError> {
1020 if self.path != ":memory:" && replica_path == Some(self.path.as_str()) {
1021 return Err(TransportError::new(
1022 "sync.invalid_request",
1023 "localAuthorityPath must name a file other than the replica's dbPath",
1024 ));
1025 }
1026 let mut authority = self.authority.lock().expect("local authority lock");
1027 *authority = None;
1029 *authority = Some(LocalAuthority::open(&self.path, schema).map_err(transport_error)?);
1030 Ok(())
1031 }
1032
1033 fn local_authority_state(&self) -> Option<&'static str> {
1034 self.authority
1035 .lock()
1036 .expect("local authority lock")
1037 .as_ref()
1038 .map(|authority| match authority.state() {
1039 AuthorityState::Local => "local",
1040 AuthorityState::Attaching => "attaching",
1041 AuthorityState::Attached => "attached",
1042 })
1043 }
1044
1045 fn attach_local_authority(&mut self) -> Result<(), TransportError> {
1046 if !self.remote_configured {
1047 return Err(unconfigured(
1048 "attachLocalAuthority needs a baseUrl for the remote server",
1049 ));
1050 }
1051 let remote = &mut self.remote;
1052 let mut authority = self.authority.lock().expect("local authority lock");
1053 authority
1054 .as_mut()
1055 .ok_or_else(|| {
1056 unconfigured("the local authority opens with create; create a client first")
1057 })?
1058 .attach(&mut |request: &[u8]| remote.sync(request).map_err(authority_error))
1059 .map_err(transport_error)
1060 }
1061
1062 fn abandon_local_authority_attach(&mut self) -> Result<(), TransportError> {
1063 if !self.remote_configured {
1064 return Err(unconfigured(
1065 "abandonLocalAuthorityAttach needs a baseUrl for the remote server",
1066 ));
1067 }
1068 let remote = &mut self.remote;
1069 let mut authority = self.authority.lock().expect("local authority lock");
1070 authority
1071 .as_mut()
1072 .ok_or_else(|| {
1073 unconfigured("the local authority opens with create; create a client first")
1074 })?
1075 .abandon_attach(Some(&mut |request: &[u8]| {
1076 remote.sync(request).map_err(authority_error)
1077 }))
1078 .map_err(transport_error)
1079 }
1080 }
1081}
1082
1083#[cfg(feature = "native-transport")]
1084mod native {
1085 use std::io::{self, Read, Write};
1097 use std::net::{TcpStream, ToSocketAddrs};
1098 use std::sync::atomic::{AtomicBool, Ordering};
1099 use std::sync::{mpsc, Arc, Condvar, Mutex};
1100 use std::thread::JoinHandle;
1101 use std::time::{Duration, Instant};
1102
1103 use polling::{Event, Events, Poller};
1104 use tungstenite::stream::MaybeTlsStream;
1105 use tungstenite::Message;
1106 use ureq::unversioned::resolver::DefaultResolver;
1107 use ureq::unversioned::transport::{
1108 time::Duration as UreqDuration, ConnectionDetails, Connector, DefaultConnector,
1109 NextTimeout, Transport as UreqTransport,
1110 };
1111 use ureq::Timeout as UreqTimeout;
1112
1113 use super::{HostTransportPolicy, Inbound, InboundBuffer, RealtimeReceiptKind, RedirectPolicy};
1114 use crate::{
1115 BlobDownload, BlobUploadGrant, RealtimeRound, RoundInbound, SegmentFetch, SegmentRequest,
1116 Transport, TransportError,
1117 };
1118
1119 struct SocketRegistration {
1121 poller: Arc<Poller>,
1122 stream: TcpStream,
1123 }
1124
1125 impl Drop for SocketRegistration {
1126 fn drop(&mut self) {
1127 let _ = self.poller.delete(&self.stream);
1128 }
1129 }
1130
1131 #[derive(Clone)]
1132 struct SocketSender {
1133 queue: mpsc::SyncSender<Outgoing>,
1134 poller: Arc<Poller>,
1135 }
1136
1137 struct Outgoing {
1138 message: Message,
1139 completed: mpsc::Sender<Result<(), TransportError>>,
1140 }
1141
1142 const ROUND_IDLE_TIMEOUT: Duration = Duration::from_secs(30);
1148 const REQUEST_CHUNK_BYTES: usize = 64 * 1024;
1151
1152 fn round_expiry(
1155 last_progress: Instant,
1156 idle: Duration,
1157 ceiling: Option<Instant>,
1158 ) -> (Instant, bool) {
1159 let silent_at = last_progress + idle;
1160 match ceiling {
1161 Some(ceiling) if ceiling < silent_at => (ceiling, true),
1162 _ => (silent_at, false),
1163 }
1164 }
1165 #[derive(Default)]
1172 pub(super) struct RoundChannel {
1173 state: Mutex<RoundState>,
1174 ready: Condvar,
1175 }
1176
1177 #[derive(Default)]
1178 struct RoundState {
1179 round: RealtimeRound,
1180 last_progress: Option<Instant>,
1183 lost: bool,
1185 outcome: Option<Result<Vec<u8>, TransportError>>,
1187 }
1188
1189 impl RoundChannel {
1190 fn begin(&self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1194 let mut state = self.state.lock().expect("round lock");
1195 state.outcome = None;
1196 let framed = state.round.begin(request)?;
1197 state.last_progress = Some(Instant::now());
1198 Ok(framed)
1199 }
1200
1201 fn progressed(&self) {
1203 self.state.lock().expect("round lock").last_progress = Some(Instant::now());
1204 }
1205
1206 fn lost(&self) -> bool {
1209 self.state.lock().expect("round lock").lost
1210 }
1211
1212 fn route_binary(&self, frame: &[u8]) -> Option<Vec<u8>> {
1216 let mut state = self.state.lock().expect("round lock");
1217 match state.round.route_binary(frame) {
1218 Ok(RoundInbound::Delta(body)) => Some(body),
1219 Ok(RoundInbound::RoundProgress) => {
1220 state.last_progress = Some(Instant::now());
1221 None
1222 }
1223 Ok(RoundInbound::Ignored) => None,
1224 Ok(RoundInbound::RoundComplete(bytes)) => {
1225 state.outcome = Some(Ok(bytes));
1226 self.ready.notify_all();
1227 None
1228 }
1229 Err(error) => {
1230 state.outcome = Some(Err(error));
1231 self.ready.notify_all();
1232 None
1233 }
1234 }
1235 }
1236
1237 fn fail_in_flight(&self, error: TransportError) {
1239 let mut state = self.state.lock().expect("round lock");
1240 state.lost = true;
1241 if state.round.in_flight() && state.outcome.is_none() {
1242 state.round.abort();
1243 state.outcome = Some(Err(error));
1244 self.ready.notify_all();
1245 }
1246 }
1247
1248 fn wait(
1251 &self,
1252 ceiling: Option<Instant>,
1253 idle: Duration,
1254 ) -> Result<Vec<u8>, TransportError> {
1255 let mut state = self.state.lock().expect("round lock");
1256 while state.outcome.is_none() {
1257 let (deadline, ceiling_binds) = round_expiry(
1258 state.last_progress.unwrap_or_else(Instant::now),
1259 idle,
1260 ceiling,
1261 );
1262 let now = Instant::now();
1263 if now >= deadline {
1264 state.round.abort();
1265 return Err(if ceiling_binds {
1266 timeout_error("sync round deadline exceeded", "round_deadline")
1267 } else {
1268 TransportError::new(
1269 "sync.transport_failed",
1270 "realtime sync round made no progress (§8.7)",
1271 )
1272 });
1273 }
1274 let (guard, _timeout) = self
1275 .ready
1276 .wait_timeout(state, deadline - now)
1277 .expect("round wait");
1278 state = guard;
1279 }
1280 state.outcome.take().expect("outcome present")
1281 }
1282 }
1283
1284 fn transfer_error(
1285 message: &'static str,
1286 cause_kind: &'static str,
1287 http_status: Option<u16>,
1288 ) -> TransportError {
1289 let mut details = serde_json::json!({ "causeKind": cause_kind });
1290 if let Some(status) = http_status {
1291 details["httpStatus"] = status.into();
1292 }
1293 let mut error = TransportError::new("transport.failed", message);
1294 error.details = Some(details);
1295 error
1296 }
1297
1298 fn timeout_error(message: &'static str, cause_kind: &'static str) -> TransportError {
1301 let mut error = TransportError::new("transport.timeout", message);
1302 error.details = Some(serde_json::json!({ "causeKind": cause_kind }));
1303 error
1304 }
1305
1306 fn redirect_error(message: &'static str, http_status: Option<u16>) -> TransportError {
1308 let mut details = serde_json::json!({ "causeKind": "redirect" });
1309 if let Some(status) = http_status {
1310 details["httpStatus"] = status.into();
1311 }
1312 let mut error = TransportError::new("transport.redirect", message);
1313 error.details = Some(details);
1314 error
1315 }
1316
1317 fn redirect_refused(resp: &ureq::http::Response<ureq::Body>) -> Result<(), TransportError> {
1322 let status = resp.status().as_u16();
1323 if (300..400).contains(&status) {
1324 return Err(redirect_error("redirect refused", Some(status)));
1325 }
1326 Ok(())
1327 }
1328
1329 fn is_timeout_io(error: &std::io::Error) -> bool {
1335 if error.kind() == std::io::ErrorKind::TimedOut {
1336 return true;
1337 }
1338 let Some(inner) = error.get_ref() else {
1339 return false;
1340 };
1341 let Some(ureq_error) = inner.downcast_ref::<ureq::Error>() else {
1342 return false;
1343 };
1344 match ureq_error {
1345 ureq::Error::Timeout(_) => true,
1346 ureq::Error::Io(nested) => is_timeout_io(nested),
1347 ureq::Error::Decompress(_, nested) => is_timeout_io(nested),
1348 _ => false,
1349 }
1350 }
1351
1352 fn http_err(message: &'static str, error: ureq::Error) -> TransportError {
1353 let (kind, status) = match error {
1354 ureq::Error::StatusCode(status) if (300..400).contains(&status) => {
1355 return redirect_error(message, Some(status))
1356 }
1357 ureq::Error::StatusCode(status) => ("status", Some(status)),
1358 ureq::Error::Timeout(_) => return timeout_error(message, "timeout"),
1359 ureq::Error::Io(error) => return io_err(message, error),
1360 ureq::Error::HostNotFound
1361 | ureq::Error::ConnectionFailed
1362 | ureq::Error::ConnectProxyFailed(_) => ("connect", None),
1363 ureq::Error::Tls(_)
1364 | ureq::Error::Rustls(_)
1365 | ureq::Error::Pem(_)
1366 | ureq::Error::TlsRequired => ("tls", None),
1367 ureq::Error::BadUri(_)
1368 | ureq::Error::Http(_)
1369 | ureq::Error::RequireHttpsOnly(_)
1370 | ureq::Error::InvalidProxyUrl => ("request", None),
1371 ureq::Error::Protocol(_) => ("protocol", None),
1372 ureq::Error::BodyExceedsLimit(_)
1373 | ureq::Error::BodyStalled
1374 | ureq::Error::Decompress(_, _) => ("body", None),
1375 ureq::Error::RedirectFailed | ureq::Error::TooManyRedirects => {
1376 return redirect_error(message, None)
1377 }
1378 _ => ("unknown", None),
1379 };
1380 transfer_error(message, kind, status)
1381 }
1382
1383 fn io_err(message: &'static str, error: std::io::Error) -> TransportError {
1384 if is_timeout_io(&error) {
1385 return timeout_error(message, "timeout");
1386 }
1387 let kind = match error.kind() {
1388 std::io::ErrorKind::ConnectionRefused
1389 | std::io::ErrorKind::ConnectionAborted
1390 | std::io::ErrorKind::ConnectionReset
1391 | std::io::ErrorKind::NotConnected => "connect",
1392 _ => "io",
1393 };
1394 transfer_error(message, kind, None)
1395 }
1396
1397 fn ws_err(message: &'static str, error: tungstenite::Error) -> TransportError {
1398 match error {
1399 tungstenite::Error::Io(error) => io_err(message, error),
1400 tungstenite::Error::Tls(_) => transfer_error(message, "tls", None),
1401 tungstenite::Error::Http(response) if response.status().is_redirection() => {
1404 redirect_error(message, Some(response.status().as_u16()))
1405 }
1406 tungstenite::Error::Http(response) => {
1407 transfer_error(message, "status", Some(response.status().as_u16()))
1408 }
1409 tungstenite::Error::Url(_) | tungstenite::Error::HttpFormat(_) => {
1410 transfer_error(message, "request", None)
1411 }
1412 _ => transfer_error(message, "protocol", None),
1413 }
1414 }
1415
1416 const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
1417
1418 fn remaining(deadline: Instant) -> io::Result<Duration> {
1419 deadline
1420 .checked_duration_since(Instant::now())
1421 .filter(|left| !left.is_zero())
1422 .ok_or_else(|| {
1423 io::Error::new(
1424 io::ErrorKind::TimedOut,
1425 "websocket establishment deadline exceeded",
1426 )
1427 })
1428 }
1429
1430 struct EstablishingStream {
1431 socket: TcpStream,
1432 deadline: Option<Instant>,
1433 }
1434
1435 impl EstablishingStream {
1436 fn timeout(&self) -> io::Result<Option<Duration>> {
1437 self.deadline.map(remaining).transpose()
1438 }
1439 fn established(&mut self) -> io::Result<TcpStream> {
1440 self.deadline = None;
1441 self.socket.set_read_timeout(None)?;
1442 self.socket.set_write_timeout(None)?;
1443 self.socket.try_clone()
1444 }
1445 }
1446
1447 fn establishment_io<T>(result: io::Result<T>, establishing: bool) -> io::Result<T> {
1448 result.map_err(|error| {
1449 if establishing
1450 && matches!(
1451 error.kind(),
1452 io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut
1453 )
1454 {
1455 io::Error::new(
1456 io::ErrorKind::TimedOut,
1457 "websocket establishment deadline exceeded",
1458 )
1459 } else {
1460 error
1461 }
1462 })
1463 }
1464
1465 impl Read for EstablishingStream {
1466 fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
1467 if self.deadline.is_some() {
1468 self.socket.set_read_timeout(self.timeout()?)?;
1469 }
1470 establishment_io(self.socket.read(bytes), self.deadline.is_some())
1471 }
1472 }
1473 impl Write for EstablishingStream {
1474 fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
1475 if self.deadline.is_some() {
1476 self.socket.set_write_timeout(self.timeout()?)?;
1477 }
1478 establishment_io(self.socket.write(bytes), self.deadline.is_some())
1479 }
1480 fn flush(&mut self) -> io::Result<()> {
1481 if self.deadline.is_some() {
1482 self.socket.set_write_timeout(self.timeout()?)?;
1483 }
1484 establishment_io(self.socket.flush(), self.deadline.is_some())
1485 }
1486 }
1487
1488 fn connect_websocket(
1489 request: tungstenite::handshake::client::Request,
1490 deadline: Instant,
1491 ) -> Result<tungstenite::WebSocket<MaybeTlsStream<EstablishingStream>>, TransportError> {
1492 let host = request
1493 .uri()
1494 .host()
1495 .ok_or_else(|| transfer_error("realtime URL invalid", "request", None))?;
1496 let host = host.trim_start_matches('[').trim_end_matches(']');
1497 let port = match request.uri().scheme_str() {
1498 Some("ws") => request.uri().port_u16().unwrap_or(80),
1499 Some("wss") => request.uri().port_u16().unwrap_or(443),
1500 _ => return Err(transfer_error("realtime URL invalid", "request", None)),
1501 };
1502 let host = host.to_owned();
1505 let (resolved, addresses) = mpsc::sync_channel(1);
1506 std::thread::Builder::new()
1507 .name("syncular-dns".to_owned())
1508 .spawn(move || {
1509 let result = (host.as_str(), port)
1510 .to_socket_addrs()
1511 .map(|items| items.collect::<Vec<_>>());
1512 let _ = resolved.send(result);
1513 })
1514 .map_err(|e| io_err("realtime DNS failed", e))?;
1515 let addresses = addresses
1516 .recv_timeout(remaining(deadline).map_err(|e| io_err("realtime DNS failed", e))?)
1517 .map_err(|e| match e {
1518 mpsc::RecvTimeoutError::Timeout => {
1519 timeout_error("realtime DNS timed out", "timeout")
1520 }
1521 mpsc::RecvTimeoutError::Disconnected => {
1522 transfer_error("realtime DNS failed", "io", None)
1523 }
1524 })?
1525 .map_err(|e| io_err("realtime DNS failed", e))?;
1526 let mut last_error = io::Error::new(io::ErrorKind::NotFound, "DNS returned no addresses");
1527 let mut socket = None;
1528 for address in addresses {
1529 match TcpStream::connect_timeout(
1530 &address,
1531 remaining(deadline).map_err(|e| io_err("realtime TCP connect failed", e))?,
1532 ) {
1533 Ok(stream) => {
1534 socket = Some(stream);
1535 break;
1536 }
1537 Err(error) => last_error = error,
1538 }
1539 }
1540 let socket = socket.ok_or_else(|| io_err("realtime TCP connect failed", last_error))?;
1541 socket
1542 .set_nodelay(true)
1543 .map_err(|e| io_err("realtime TCP connect failed", e))?;
1544 let stream = EstablishingStream {
1545 socket,
1546 deadline: Some(deadline),
1547 };
1548 let (ws, _) = tungstenite::client_tls_with_config(request, stream, None, None).map_err(
1551 |e| match e {
1552 tungstenite::HandshakeError::Failure(error) => {
1553 ws_err("realtime connection failed", error)
1554 }
1555 tungstenite::HandshakeError::Interrupted(_) => {
1558 transfer_error("realtime connection failed", "protocol", None)
1559 }
1560 },
1561 )?;
1562 remaining(deadline).map_err(|e| io_err("realtime connection failed", e))?;
1563 Ok(ws)
1564 }
1565
1566 #[cfg(test)]
1567 mod establishment_tests {
1568 use super::*;
1569 use std::net::{Shutdown, TcpListener};
1570 use tungstenite::client::IntoClientRequest;
1571
1572 fn withheld_response(tls: bool, old: bool) {
1575 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1576 let url = format!(
1577 "{}://{}/realtime",
1578 if tls { "wss" } else { "ws" },
1579 listener.local_addr().unwrap()
1580 );
1581 let (accepted_tx, accepted_rx) = mpsc::channel();
1582 let (release_tx, release_rx) = mpsc::channel();
1583 let peer = std::thread::spawn(move || {
1584 let (mut stream, _) = listener.accept().unwrap();
1585 stream
1586 .set_read_timeout(Some(Duration::from_secs(2)))
1587 .unwrap();
1588 let mut bytes = [0; 4096];
1589 assert!(stream.read(&mut bytes).unwrap() > 0);
1590 accepted_tx.send(()).unwrap();
1591 release_rx.recv_timeout(Duration::from_secs(3)).unwrap();
1592 let _ = stream.shutdown(Shutdown::Both);
1593 });
1594 let (done_tx, done_rx) = mpsc::channel();
1595 let worker = std::thread::spawn(move || {
1596 let result = if old {
1597 tungstenite::connect(url)
1598 .map(|_| ())
1599 .map_err(|e| e.to_string())
1600 } else {
1601 connect_websocket(
1602 url.into_client_request().unwrap(),
1603 Instant::now() + Duration::from_millis(250),
1604 )
1605 .map(|_| ())
1606 .map_err(|e| format!("{}: {}", e.code, e.message))
1607 };
1608 done_tx.send(result).unwrap();
1609 });
1610 accepted_rx.recv_timeout(Duration::from_secs(2)).unwrap();
1611 if old {
1612 assert!(
1613 matches!(
1614 done_rx.recv_timeout(Duration::from_millis(500)),
1615 Err(mpsc::RecvTimeoutError::Timeout)
1616 ),
1617 "original connect unexpectedly had a deadline"
1618 );
1619 release_tx.send(()).unwrap();
1620 assert!(done_rx
1621 .recv_timeout(Duration::from_secs(2))
1622 .unwrap()
1623 .is_err());
1624 } else {
1625 let error = done_rx
1626 .recv_timeout(Duration::from_secs(2))
1627 .expect("establishment remained blocked")
1628 .unwrap_err();
1629 assert_eq!(error, "transport.timeout: realtime connection failed");
1630 eprintln!(
1631 "withheld {} produces {error}",
1632 if tls { "TLS" } else { "upgrade" }
1633 );
1634 release_tx.send(()).unwrap();
1635 }
1636 worker.join().unwrap();
1637 peer.join().unwrap();
1638 }
1639
1640 #[test]
1641 fn original_connect_blocks_on_withheld_upgrade_and_tls() {
1642 withheld_response(false, true);
1643 withheld_response(true, true);
1644 }
1645 #[test]
1646 fn establishment_deadline_errors_on_withheld_upgrade_and_tls() {
1647 withheld_response(false, false);
1648 withheld_response(true, false);
1649 }
1650
1651 #[test]
1652 fn slow_upgrade_bytes_cannot_renew_the_absolute_deadline() {
1653 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1654 let url = format!("ws://{}", listener.local_addr().unwrap());
1655 let peer = std::thread::spawn(move || {
1656 let (mut stream, _) = listener.accept().unwrap();
1657 let mut bytes = [0; 4096];
1658 let _ = stream.read(&mut bytes).unwrap();
1659 for byte in b"HTTP/1.1 101 Switching Protocols\r\n" {
1660 if stream.write_all(&[*byte]).is_err() {
1661 break;
1662 }
1663 std::thread::sleep(Duration::from_millis(40));
1664 }
1665 });
1666 let started = Instant::now();
1667 assert!(connect_websocket(
1668 url.into_client_request().unwrap(),
1669 started + Duration::from_millis(250)
1670 )
1671 .is_err());
1672 assert!(started.elapsed() < Duration::from_secs(1));
1673 peer.join().unwrap();
1674 }
1675
1676 #[test]
1677 fn saturated_tcp_accept_queue_returns_an_observable_error() {
1678 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1681 let address = listener.local_addr().unwrap();
1682 let mut queued = Vec::new();
1683 loop {
1684 match TcpStream::connect_timeout(&address, Duration::from_millis(50)) {
1685 Ok(stream) => queued.push(stream),
1686 Err(error) => {
1687 assert!(
1688 matches!(
1689 error.kind(),
1690 io::ErrorKind::TimedOut
1691 | io::ErrorKind::WouldBlock
1692 | io::ErrorKind::ConnectionRefused
1693 | io::ErrorKind::ConnectionReset
1694 ),
1695 "{error:?}"
1696 );
1697 break;
1698 }
1699 }
1700 assert!(queued.len() < 4096, "listener queue did not saturate");
1701 }
1702 let started = Instant::now();
1703 let url = format!("ws://{address}");
1704 let error = match connect_websocket(
1705 url.into_client_request().unwrap(),
1706 started + Duration::from_millis(250),
1707 ) {
1708 Err(error) => error,
1709 Ok(_) => panic!("full accept queue unexpectedly connected"),
1710 };
1711 assert!(
1712 (error.code == "transport.timeout" || error.code == "transport.failed")
1713 && (error.message == "realtime TCP connect failed"
1714 || error.message == "realtime connection failed"),
1715 "{}: {}",
1716 error.code,
1717 error.message
1718 );
1719 assert!(started.elapsed() < Duration::from_secs(2));
1720 }
1721 }
1722
1723 fn segment_error(mut error: TransportError) -> TransportError {
1727 if error.code == "transport.failed" {
1728 error.code = "sync.transport_failed".into();
1729 error.message = "segment transfer failed".into();
1730 }
1731 error
1732 }
1733
1734 fn is_would_block(e: &tungstenite::Error) -> bool {
1736 matches!(
1737 e,
1738 tungstenite::Error::Io(io) if matches!(
1739 io.kind(),
1740 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
1741 )
1742 )
1743 }
1744
1745 pub struct NativeTransport {
1746 base_url: String,
1747 ws_url: String,
1748 headers: Vec<(String, String)>,
1750 base_url_has_userinfo: bool,
1754 agent: ureq::Agent,
1755 policy: HostTransportPolicy,
1756 round_deadline_at: Option<std::time::Instant>,
1760 pub signed_urls: bool,
1761 pub inbound: Arc<InboundBuffer>,
1762 outgoing: Option<Box<SocketSender>>,
1764 reader: Option<JoinHandle<()>>,
1765 reader_stop: Arc<AtomicBool>,
1766 round: Arc<RoundChannel>,
1768 realtime_client_id: Option<String>,
1769 }
1770
1771 #[derive(Debug)]
1778 struct IdleConnector(Option<Duration>);
1779
1780 impl Connector<Box<dyn UreqTransport>> for IdleConnector {
1781 type Out = Box<dyn UreqTransport>;
1782
1783 fn connect(
1784 &self,
1785 _: &ConnectionDetails,
1786 chained: Option<Box<dyn UreqTransport>>,
1787 ) -> Result<Option<Self::Out>, ureq::Error> {
1788 Ok(chained.map(|inner| match self.0 {
1789 Some(idle) => IdleTransport { inner, idle }.boxed(),
1790 None => inner,
1791 }))
1792 }
1793 }
1794
1795 #[derive(Debug)]
1796 struct IdleTransport {
1797 inner: Box<dyn UreqTransport>,
1798 idle: Duration,
1799 }
1800
1801 fn idle_bound(mut timeout: NextTimeout, idle: Duration) -> NextTimeout {
1806 let idle = UreqDuration::Exact(idle);
1807 if timeout.after.is_not_happening() || timeout.after.as_nanos() > idle.as_nanos() {
1808 timeout.after = idle;
1809 timeout.reason = UreqTimeout::RecvBody;
1810 }
1811 timeout
1812 }
1813
1814 impl UreqTransport for IdleTransport {
1815 fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers {
1816 self.inner.buffers()
1817 }
1818 fn transmit_output(
1819 &mut self,
1820 amount: usize,
1821 timeout: NextTimeout,
1822 ) -> Result<(), ureq::Error> {
1823 self.inner
1824 .transmit_output(amount, idle_bound(timeout, self.idle))
1825 }
1826 fn await_input(&mut self, timeout: NextTimeout) -> Result<bool, ureq::Error> {
1827 self.inner.await_input(idle_bound(timeout, self.idle))
1828 }
1829 fn is_open(&mut self) -> bool {
1830 self.inner.is_open()
1831 }
1832 fn is_tls(&self) -> bool {
1833 self.inner.is_tls()
1834 }
1835 }
1836
1837 fn build_agent(policy: &HostTransportPolicy) -> ureq::Agent {
1838 let config = ureq::Agent::config_builder()
1839 .timeout_resolve(policy.request_timeout)
1840 .timeout_connect(policy.request_timeout)
1841 .timeout_send_request(policy.request_timeout)
1842 .timeout_recv_response(policy.request_timeout)
1843 .max_redirects(match policy.redirects {
1844 RedirectPolicy::Deny => 0,
1845 RedirectPolicy::Follow => 10,
1846 })
1847 .max_redirects_will_error(matches!(policy.redirects, RedirectPolicy::Follow))
1848 .redirect_auth_headers(ureq::config::RedirectAuthHeaders::Never)
1849 .http_status_as_error(false)
1854 .build();
1855 ureq::Agent::with_parts(
1856 config,
1857 DefaultConnector::new().chain(IdleConnector(policy.request_timeout)),
1858 DefaultResolver::default(),
1859 )
1860 }
1861
1862 fn derive_ws_url(base_url: &str) -> String {
1863 let ws = if let Some(rest) = base_url.strip_prefix("https://") {
1866 format!("wss://{rest}")
1867 } else if let Some(rest) = base_url.strip_prefix("http://") {
1868 format!("ws://{rest}")
1869 } else {
1870 base_url.to_owned()
1871 };
1872 let trimmed = ws.trim_end_matches('/');
1873 format!("{trimmed}/realtime")
1874 }
1875
1876 impl NativeTransport {
1877 pub fn fork_round(&self) -> Self {
1878 Self {
1879 base_url: self.base_url.clone(),
1880 ws_url: self.ws_url.clone(),
1881 headers: self.headers.clone(),
1882 base_url_has_userinfo: self.base_url_has_userinfo,
1883 agent: self.agent.clone(),
1884 policy: self.policy.clone(),
1885 round_deadline_at: None,
1886 signed_urls: self.signed_urls,
1887 inbound: self.inbound.clone(),
1888 outgoing: self.outgoing.clone(),
1889 reader: None,
1890 reader_stop: self.reader_stop.clone(),
1891 round: self.round.clone(),
1892 realtime_client_id: self.realtime_client_id.clone(),
1893 }
1894 }
1895
1896 pub fn new(
1897 base_url: &str,
1898 config: &serde_json::Value,
1899 notify: Option<Arc<dyn Fn() + Send + Sync>>,
1900 ) -> Result<Self, String> {
1901 let policy = HostTransportPolicy::from_config(config)?;
1902 Self::with_policy(base_url, config, policy, notify)
1903 }
1904
1905 pub fn with_policy(
1906 base_url: &str,
1907 config: &serde_json::Value,
1908 policy: HostTransportPolicy,
1909 notify: Option<Arc<dyn Fn() + Send + Sync>>,
1910 ) -> Result<Self, String> {
1911 policy.validate()?;
1912 let mut headers = Vec::new();
1913 if let Some(map) = config.get("headers").and_then(|v| v.as_object()) {
1914 for (k, v) in map {
1915 if let Some(s) = v.as_str() {
1916 headers.push((k.clone(), s.to_owned()));
1917 }
1918 }
1919 }
1920 let ws_url = config
1921 .get("wsUrl")
1922 .and_then(|v| v.as_str())
1923 .map(str::to_owned)
1924 .unwrap_or_else(|| derive_ws_url(base_url));
1925 let base_url_has_userinfo = url::Url::parse(base_url)
1928 .map(|url| !url.username().is_empty() || url.password().is_some())
1929 .unwrap_or(false);
1930 Ok(NativeTransport {
1931 base_url: base_url.trim_end_matches('/').to_owned(),
1932 ws_url,
1933 headers,
1934 base_url_has_userinfo,
1935 agent: build_agent(&policy),
1936 policy,
1937 round_deadline_at: None,
1938 signed_urls: false,
1939 inbound: Arc::new(match notify {
1940 Some(notify) => InboundBuffer::with_notify(notify),
1941 None => InboundBuffer::default(),
1942 }),
1943 outgoing: None,
1944 reader: None,
1945 reader_stop: Arc::new(AtomicBool::new(false)),
1946 round: Arc::new(RoundChannel::default()),
1947 realtime_client_id: None,
1948 })
1949 }
1950
1951 pub fn set_policy(&mut self, policy: HostTransportPolicy) -> Result<(), String> {
1955 policy.validate()?;
1956 self.agent = build_agent(&policy);
1957 self.policy = policy;
1958 Ok(())
1959 }
1960
1961 pub fn round_deadline(&self) -> Option<Duration> {
1962 self.policy.round_deadline
1963 }
1964
1965 pub fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
1966 self.round_deadline_at = deadline;
1967 }
1968
1969 fn round_remaining(&self) -> Result<Option<Duration>, TransportError> {
1974 match self.round_deadline_at {
1975 Some(deadline) => deadline
1976 .checked_duration_since(std::time::Instant::now())
1977 .map(Some)
1978 .ok_or_else(|| timeout_error("sync round deadline exceeded", "round_deadline")),
1979 None => Ok(None),
1980 }
1981 }
1982
1983 fn credentialed(&self) -> bool {
1986 !self.headers.is_empty() || self.base_url_has_userinfo
1987 }
1988
1989 fn finish_request<B>(
1995 &self,
1996 req: ureq::RequestBuilder<B>,
1997 credentialed: bool,
1998 ) -> Result<ureq::RequestBuilder<B>, TransportError> {
1999 let timeout = self.round_remaining()?;
2000 let follow = self.policy.redirects == RedirectPolicy::Follow && !credentialed;
2001 Ok(req
2002 .config()
2003 .timeout_global(timeout)
2004 .max_redirects(if follow { 10 } else { 0 })
2005 .max_redirects_will_error(true)
2006 .build())
2007 }
2008
2009 fn check_request_bytes(&self, len: u64) -> Result<(), TransportError> {
2011 match self.policy.max_request_bytes {
2012 Some(max) if len > max => {
2013 let mut error = TransportError::new(
2014 "transport.request_too_large",
2015 "request body exceeds the configured transport limit",
2016 );
2017 error.details = Some(serde_json::json!({ "limit": max, "size": len }));
2018 Err(error)
2019 }
2020 _ => Ok(()),
2021 }
2022 }
2023
2024 fn post_sync(&self, path: &str, body: &[u8]) -> Result<Vec<u8>, TransportError> {
2025 self.check_request_bytes(body.len() as u64)?;
2026 let credentialed = self.credentialed();
2027 let url = format!("{}{}", self.base_url, path);
2028 let mut req = self
2031 .agent
2032 .post(&url)
2033 .header("content-type", "application/vnd.syncular.sync.v2");
2034 for (k, v) in &self.headers {
2035 req = req.header(k.as_str(), v.as_str());
2036 }
2037 let req = self.finish_request(req, credentialed)?;
2038 let resp = req
2039 .send(body)
2040 .map_err(|e| http_err("sync request failed", e))?;
2041 read_body(resp, None, self.policy.max_response_bytes, None)
2042 }
2043
2044 fn post_operation(&self, body: &[u8]) -> Result<Vec<u8>, TransportError> {
2045 self.check_request_bytes(body.len() as u64)?;
2046 let credentialed = self.credentialed();
2047 let url = format!("{}/operations", self.base_url);
2048 let mut req = self.agent.post(&url).header(
2049 "content-type",
2050 "application/vnd.syncular.operations.v1+json",
2051 );
2052 for (key, value) in &self.headers {
2053 req = req.header(key.as_str(), value.as_str());
2054 }
2055 let req = self.finish_request(req, credentialed)?;
2056 let response = req
2057 .send(body)
2058 .map_err(|error| http_err("remote operation request failed", error))?;
2059 read_body(response, None, self.policy.max_response_bytes, None)
2060 }
2061
2062 pub fn set_headers(&mut self, headers: Vec<(String, String)>) {
2063 self.headers = headers;
2064 }
2065
2066 pub fn set_realtime_client_id(&mut self, client_id: Option<&str>) {
2067 self.realtime_client_id = client_id.map(str::to_owned);
2068 }
2069
2070 pub fn shutdown(&mut self) {
2071 self.reader_stop.store(true, Ordering::SeqCst);
2072 self.round.fail_in_flight(TransportError::new(
2075 "sync.transport_failed",
2076 "realtime disconnected mid-round (§8.7)",
2077 ));
2078 if let Some(outgoing) = &self.outgoing {
2079 let _ = outgoing.poller.notify();
2080 }
2081 if let Some(handle) = self.reader.take() {
2082 let _ = handle.join();
2083 }
2084 self.outgoing = None;
2085 }
2086
2087 fn send_message(&mut self, message: Message) -> Result<(), TransportError> {
2088 let (wait, round_bound) = match self.round_deadline_at {
2093 Some(deadline) => {
2094 match deadline.checked_duration_since(std::time::Instant::now()) {
2095 Some(remaining) if remaining < ROUND_IDLE_TIMEOUT => (remaining, true),
2096 Some(_) => (ROUND_IDLE_TIMEOUT, false),
2097 None => {
2100 return Err(timeout_error("realtime send timed out", "round_deadline"));
2101 }
2102 }
2103 }
2104 None => (ROUND_IDLE_TIMEOUT, false),
2105 };
2106 let result = (|| {
2107 let Some(outgoing) = &self.outgoing else {
2108 return Err(TransportError::new(
2109 "transport.failed",
2110 "realtime not connected",
2111 ));
2112 };
2113 let (completed, result) = mpsc::channel();
2114 outgoing
2115 .queue
2116 .try_send(Outgoing { message, completed })
2117 .map_err(|_| {
2118 TransportError::new("transport.failed", "realtime send queue unavailable")
2119 })?;
2120 outgoing
2121 .poller
2122 .notify()
2123 .map_err(|e| io_err("realtime wake failed", e))?;
2124 result.recv_timeout(wait).map_err(|_| {
2125 if round_bound {
2126 timeout_error("realtime send timed out", "round_deadline")
2127 } else {
2128 TransportError::new("transport.failed", "realtime send did not complete")
2129 }
2130 })?
2131 })();
2132 if let Err(error) = &result {
2135 self.round.fail_in_flight(error.clone());
2139 self.shutdown();
2140 self.inbound.push(Inbound::Disconnected(error.clone()));
2143 }
2144 result
2145 }
2146 }
2147
2148 fn http_status_body_error(status: u16, body: &[u8]) -> TransportError {
2152 let mut error = TransportError::new("transport.failed", "HTTP request failed");
2153 let mut details = serde_json::Map::new();
2154 if let Ok(serde_json::Value::Object(parsed)) =
2155 serde_json::from_slice::<serde_json::Value>(body)
2156 {
2157 if let Some(code) = parsed.get("code").and_then(serde_json::Value::as_str) {
2158 if !code.is_empty() {
2159 error.code = code.to_owned();
2160 }
2161 }
2162 if let Some(message) = parsed.get("message").and_then(serde_json::Value::as_str) {
2163 error.message = message.to_owned();
2164 }
2165 for key in ["category", "recommendedAction"] {
2166 if let Some(value) = parsed.get(key).and_then(serde_json::Value::as_str) {
2167 details.insert(key.to_owned(), serde_json::Value::from(value));
2168 }
2169 }
2170 if let Some(retryable) = parsed.get("retryable").and_then(serde_json::Value::as_bool) {
2171 details.insert("retryable".to_owned(), serde_json::Value::from(retryable));
2172 }
2173 if let Some(serde_json::Value::Object(inner)) = parsed.get("details") {
2174 for (key, value) in inner {
2175 details.insert(key.clone(), value.clone());
2176 }
2177 }
2178 }
2179 details.insert("causeKind".to_owned(), serde_json::Value::from("status"));
2180 details.insert("httpStatus".to_owned(), serde_json::Value::from(status));
2181 error.details = Some(serde_json::Value::Object(details));
2182 error
2183 }
2184
2185 fn read_body(
2190 resp: ureq::http::Response<ureq::Body>,
2191 mut progress: Option<&mut dyn FnMut(u64)>,
2192 max: Option<u64>,
2193 stop: Option<u64>,
2194 ) -> Result<Vec<u8>, TransportError> {
2195 redirect_refused(&resp)?;
2196 use std::io::Read;
2197 let status = resp.status().as_u16();
2198 let mut reader = resp.into_body().into_with_config().reader();
2199 let mut bytes = Vec::new();
2200 let mut chunk = [0u8; 64 * 1024];
2201 let mut reported = 0;
2202 loop {
2203 let n = reader.read(&mut chunk).map_err(|error| {
2204 if is_timeout_io(&error) {
2205 timeout_error("response body read times out", "timeout")
2206 } else {
2207 transfer_error("response body read failed", "body", Some(status))
2208 }
2209 })?;
2210 if n == 0 {
2211 break;
2212 }
2213 if let Some(max) = max {
2216 if bytes.len() as u64 + n as u64 > max {
2217 return Err(response_too_large(max));
2218 }
2219 }
2220 bytes.extend_from_slice(&chunk[..n]);
2221 if stop.is_some_and(|stop| bytes.len() as u64 > stop) {
2222 break;
2223 }
2224 if bytes.len() - reported >= 64 * 1024 {
2225 if let Some(callback) = progress.as_mut() {
2226 callback(bytes.len() as u64);
2227 }
2228 reported = bytes.len();
2229 }
2230 }
2231 if bytes.len() != reported {
2232 if let Some(callback) = progress {
2233 callback(bytes.len() as u64);
2234 }
2235 }
2236 if status >= 400 {
2237 return Err(http_status_body_error(status, &bytes));
2238 }
2239 Ok(bytes)
2240 }
2241
2242 fn response_too_large(max: u64) -> TransportError {
2243 let mut error = TransportError::new(
2244 "transport.response_too_large",
2245 "response body exceeds the configured transport limit",
2246 );
2247 error.details = Some(serde_json::json!({ "limit": max }));
2248 error
2249 }
2250
2251 impl Transport for NativeTransport {
2252 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2253 self.post_sync("/sync", request)
2254 }
2255
2256 fn remote_operation(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2257 self.post_operation(request)
2258 }
2259
2260 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2261 if self.outgoing.is_none() {
2271 return self.post_sync("/sync", request);
2272 }
2273 self.round_remaining()?;
2274 let framed = self.round.begin(request)?;
2275 let (tag, body) = framed.split_first().expect("framed request carries a tag");
2276 for chunk in body.chunks(REQUEST_CHUNK_BYTES) {
2277 let mut message = Vec::with_capacity(chunk.len() + 1);
2278 message.push(*tag);
2279 message.extend_from_slice(chunk);
2280 if let Err(error) = self.send_message(Message::Binary(message.into())) {
2281 self.round.fail_in_flight(error);
2282 break;
2283 }
2284 self.round.progressed();
2285 }
2286 let outcome = self.round.wait(self.round_deadline_at, ROUND_IDLE_TIMEOUT);
2287 if let Err(error) = &outcome {
2292 if !self.round.lost() {
2293 self.shutdown();
2294 self.inbound.push(Inbound::Disconnected(error.clone()));
2295 }
2296 }
2297 outcome
2298 }
2299
2300 fn download_segment(
2301 &mut self,
2302 request: &SegmentRequest,
2303 on_progress: &mut dyn FnMut(u64),
2304 ) -> Result<Vec<u8>, TransportError> {
2305 let url = format!("{}/segments/{}", self.base_url, request.segment_id);
2312 let credentialed = self.credentialed();
2313 let mut req = self
2314 .agent
2315 .get(&url)
2316 .header("x-syncular-scopes", &request.requested_scopes_json);
2317 for (k, v) in &self.headers {
2318 req = req.header(k.as_str(), v.as_str());
2319 }
2320 let req = self.finish_request(req, credentialed)?;
2321 let resp = req
2322 .call()
2323 .map_err(|error| segment_error(http_err("segment request failed", error)))?;
2324 read_body(
2325 resp,
2326 Some(on_progress),
2327 self.policy.max_response_bytes,
2328 Some(request.byte_length),
2329 )
2330 .map_err(segment_error)
2331 }
2332
2333 fn supports_url_fetch(&self) -> bool {
2334 self.signed_urls
2335 }
2336
2337 fn download_segments(
2338 &mut self,
2339 fetches: &[SegmentFetch],
2340 on_progress: &mut (dyn FnMut(usize, u64) + Send),
2341 ) -> Vec<Result<Vec<u8>, TransportError>> {
2342 let workers = (0..fetches.len().min(4))
2351 .map(|_| {
2352 let mut worker = self.fork_round();
2353 worker.round_deadline_at = self.round_deadline_at;
2354 worker
2355 })
2356 .collect::<Vec<_>>();
2357 let slots = fetches.iter().map(|_| Mutex::new(None)).collect::<Vec<_>>();
2358 let progress = Mutex::new(on_progress);
2359 let next = std::sync::atomic::AtomicUsize::new(0);
2360 std::thread::scope(|scope| {
2361 for worker in workers {
2362 let (slots, progress, next) = (&slots, &progress, &next);
2363 scope.spawn(move || {
2364 let mut worker = worker;
2365 loop {
2366 let index = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2367 let Some(fetch) = fetches.get(index) else {
2368 break;
2369 };
2370 let mut report = |bytes: u64| {
2371 (*progress.lock().expect("segment progress lock"))(index, bytes);
2372 };
2373 let result = match fetch {
2374 SegmentFetch::Direct(request) => {
2375 worker.download_segment(request, &mut report)
2376 }
2377 SegmentFetch::Url { url, byte_length } => {
2378 worker.fetch_url(url, *byte_length, &mut report)
2379 }
2380 };
2381 *slots[index].lock().expect("segment result slot") = Some(result);
2382 }
2383 });
2384 }
2385 });
2386 slots
2387 .into_iter()
2388 .map(|slot| slot.into_inner().expect("segment result slot"))
2389 .map(|result| result.expect("every fetch is attempted"))
2390 .collect()
2391 }
2392
2393 fn fetch_url(
2394 &mut self,
2395 url: &str,
2396 byte_length: u64,
2397 on_progress: &mut dyn FnMut(u64),
2398 ) -> Result<Vec<u8>, TransportError> {
2399 let req = self.finish_request(self.agent.get(url), true)?;
2401 let resp = req
2402 .call()
2403 .map_err(|error| segment_error(http_err("segment request failed", error)))?;
2404 read_body(
2405 resp,
2406 Some(on_progress),
2407 self.policy.max_response_bytes,
2408 Some(byte_length),
2409 )
2410 .map_err(segment_error)
2411 }
2412
2413 fn blob_upload(
2414 &mut self,
2415 blob_id: &str,
2416 bytes: &[u8],
2417 media_type: Option<&str>,
2418 ) -> Result<(), TransportError> {
2419 self.check_request_bytes(bytes.len() as u64)?;
2422 let credentialed = self.credentialed();
2423 let url = format!("{}/blobs/{}", self.base_url, blob_id);
2424 let mut req = self.agent.put(&url).header(
2425 "content-type",
2426 media_type.unwrap_or("application/octet-stream"),
2427 );
2428 for (k, v) in &self.headers {
2429 req = req.header(k.as_str(), v.as_str());
2430 }
2431 let req = self.finish_request(req, credentialed)?;
2432 let resp = req
2433 .send(bytes)
2434 .map_err(|e| http_err("blob upload failed", e))?;
2435 read_body(resp, None, self.policy.max_response_bytes, None)?;
2436 Ok(())
2437 }
2438
2439 fn blob_download(
2440 &mut self,
2441 blob_id: &str,
2442 byte_length: Option<u64>,
2443 ) -> Result<BlobDownload, TransportError> {
2444 let credentialed = self.credentialed();
2447 let url = format!("{}/blobs/{}", self.base_url, blob_id);
2448 let mut req = self.agent.get(&url);
2449 for (k, v) in &self.headers {
2450 req = req.header(k.as_str(), v.as_str());
2451 }
2452 let req = self.finish_request(req, credentialed)?;
2453 let resp = req.call().map_err(|cause| {
2454 let status = match &cause {
2455 ureq::Error::StatusCode(status) => Some(*status),
2456 _ => None,
2457 };
2458 let mut error = http_err("blob download failed", cause);
2459 match status {
2460 Some(404) => {
2461 error.code = "blob.not_found".into();
2462 error.message = "blob not found".into();
2463 }
2464 Some(403) => {
2465 error.code = "blob.forbidden".into();
2466 error.message = "blob download forbidden".into();
2467 }
2468 Some(401) => {
2469 error.code = "sync.auth_required".into();
2470 error.message = "host authentication required".into();
2471 }
2472 _ => {}
2473 }
2474 error
2475 })?;
2476 let is_json = resp
2479 .headers()
2480 .get(ureq::http::header::CONTENT_TYPE)
2481 .and_then(|value| value.to_str().ok())
2482 .is_some_and(|value| value.contains("application/json"));
2483 let stop = byte_length.filter(|_| !is_json);
2486 let body = read_body(resp, None, self.policy.max_response_bytes, stop).map_err(
2487 |mut error| {
2488 let status = error
2492 .details
2493 .as_ref()
2494 .filter(|details| {
2495 details.get("causeKind").and_then(serde_json::Value::as_str)
2496 == Some("status")
2497 })
2498 .and_then(|details| details.get("httpStatus"))
2499 .and_then(serde_json::Value::as_u64);
2500 match status {
2501 Some(404) => {
2502 error.code = "blob.not_found".into();
2503 error.message = "blob not found".into();
2504 }
2505 Some(403) => {
2506 error.code = "blob.forbidden".into();
2507 error.message = "blob download forbidden".into();
2508 }
2509 Some(401) => {
2510 error.code = "sync.auth_required".into();
2511 error.message = "host authentication required".into();
2512 }
2513 Some(_) => {
2514 error.code = "transport.failed".into();
2515 error.message = "blob download failed".into();
2516 }
2517 None => {}
2518 }
2519 error
2520 },
2521 )?;
2522 if is_json {
2523 if let Ok(parsed) = serde_json::from_slice::<serde_json::Value>(&body) {
2524 if let Some(u) = parsed.get("url").and_then(|v| v.as_str()) {
2525 return Ok(BlobDownload::Url {
2526 url: u.to_owned(),
2527 url_expires_at_ms: parsed
2528 .get("urlExpiresAtMs")
2529 .and_then(|v| v.as_i64()),
2530 });
2531 }
2532 }
2533 }
2534 Ok(BlobDownload::Bytes(body))
2535 }
2536
2537 fn fetch_blob_url(
2538 &mut self,
2539 url: &str,
2540 byte_length: Option<u64>,
2541 ) -> Result<Vec<u8>, TransportError> {
2542 let req = self.finish_request(self.agent.get(url), true)?;
2545 let resp = req.call().map_err(|e| http_err("URL fetch failed", e))?;
2546 read_body(resp, None, self.policy.max_response_bytes, byte_length)
2547 }
2548
2549 fn blob_upload_grant(
2550 &mut self,
2551 blob_id: &str,
2552 byte_length: u64,
2553 media_type: Option<&str>,
2554 ) -> Result<BlobUploadGrant, TransportError> {
2555 let url = format!("{}/blobs/{}/upload-grant", self.base_url, blob_id);
2556 let body = serde_json::json!({
2557 "byteLength": byte_length,
2558 "mediaType": media_type,
2559 })
2560 .to_string();
2561 self.check_request_bytes(body.len() as u64)?;
2562 let credentialed = self.credentialed();
2563 let mut req = self
2564 .agent
2565 .post(&url)
2566 .header("content-type", "application/json");
2567 for (k, v) in &self.headers {
2568 req = req.header(k.as_str(), v.as_str());
2569 }
2570 let req = self.finish_request(req, credentialed)?;
2571 let resp = req
2572 .send(body)
2573 .map_err(|e| http_err("blob upload grant request failed", e))?;
2574 let grant_body = read_body(resp, None, self.policy.max_response_bytes, None)?;
2575 let parsed: serde_json::Value = serde_json::from_slice(&grant_body)
2576 .map_err(|_| transfer_error("blob upload grant decode failed", "decode", None))?;
2577 if let Some(u) = parsed.get("url").and_then(|v| v.as_str()) {
2578 return Ok(BlobUploadGrant::Url {
2579 url: u.to_owned(),
2580 url_expires_at_ms: parsed.get("urlExpiresAtMs").and_then(|v| v.as_i64()),
2581 });
2582 }
2583 if parsed.get("present").and_then(|v| v.as_bool()) == Some(true) {
2584 return Ok(BlobUploadGrant::Present);
2585 }
2586 Ok(BlobUploadGrant::None)
2587 }
2588
2589 fn blob_put_url(
2590 &mut self,
2591 blob_id: &str,
2592 url: &str,
2593 bytes: &[u8],
2594 media_type: Option<&str>,
2595 ) -> Result<(), TransportError> {
2596 let digest = blob_id
2601 .strip_prefix("sha256:")
2602 .filter(|hex| {
2603 hex.len() == 64
2604 && hex
2605 .bytes()
2606 .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
2607 })
2608 .map(|hex| {
2609 (0..64)
2610 .step_by(2)
2611 .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).expect("hex digits"))
2612 .collect::<Vec<u8>>()
2613 })
2614 .ok_or_else(|| {
2615 TransportError::new(
2616 "sync.invalid_request",
2617 "the blob id is not a sha256 blob id (§5.9.1)",
2618 )
2619 })?;
2620 self.check_request_bytes(bytes.len() as u64)?;
2621 let req = self.finish_request(
2622 self.agent
2623 .put(url)
2624 .header(
2625 "content-type",
2626 media_type.unwrap_or("application/octet-stream"),
2627 )
2628 .header("x-amz-checksum-sha256", ssp2::util::base64(&digest)),
2629 true,
2630 )?;
2631 let resp = req
2632 .send(bytes)
2633 .map_err(|e| http_err("signed blob upload failed", e))?;
2634 read_body(resp, None, self.policy.max_response_bytes, None)?;
2635 Ok(())
2636 }
2637
2638 fn realtime_connect(&mut self) -> Result<(), TransportError> {
2639 if self.outgoing.is_some() {
2640 if self.reader_stop.load(Ordering::SeqCst)
2641 || self
2642 .reader
2643 .as_ref()
2644 .is_some_and(|reader| reader.is_finished())
2645 {
2646 self.shutdown();
2647 } else {
2648 return Ok(());
2649 }
2650 }
2651 use tungstenite::client::IntoClientRequest;
2657 let mut url = url::Url::parse(&self.ws_url)
2658 .map_err(|_| transfer_error("realtime URL invalid", "request", None))?;
2659 if let Some(client_id) = self.realtime_client_id.as_deref() {
2660 let retained_query: Vec<(String, String)> = url
2661 .query_pairs()
2662 .filter(|(key, _)| key != "clientId")
2663 .map(|(key, value)| (key.into_owned(), value.into_owned()))
2664 .collect();
2665 url.set_query(None);
2666 url.query_pairs_mut()
2667 .extend_pairs(retained_query)
2668 .append_pair("clientId", client_id);
2669 }
2670 let mut request = url
2671 .as_str()
2672 .into_client_request()
2673 .map_err(|error| ws_err("realtime request failed", error))?;
2674 {
2675 let out = request.headers_mut();
2676 for (k, v) in &self.headers {
2677 if let (Ok(name), Ok(value)) = (
2678 tungstenite::http::HeaderName::try_from(k.as_str()),
2679 tungstenite::http::HeaderValue::try_from(v.as_str()),
2680 ) {
2681 out.insert(name, value);
2682 }
2683 }
2684 }
2685 if self.policy.redirects == RedirectPolicy::Follow {
2688 return Err(redirect_error(
2689 "realtime establishment does not follow redirects",
2690 None,
2691 ));
2692 }
2693 let mut ws = connect_websocket(request, Instant::now() + CONNECT_TIMEOUT)?;
2694 let stream = match ws.get_mut() {
2695 MaybeTlsStream::Plain(stream) => stream.established(),
2696 MaybeTlsStream::Rustls(stream) => stream.get_mut().established(),
2697 _ => {
2698 return Err(TransportError::new(
2699 "transport.failed",
2700 "unsupported websocket stream",
2701 ))
2702 }
2703 }
2704 .map_err(|e| io_err("realtime clone failed", e))?;
2705 stream
2706 .set_nonblocking(true)
2707 .map_err(|e| io_err("realtime nonblocking failed", e))?;
2708 let poller = Arc::new(Poller::new().map_err(|e| io_err("realtime poller failed", e))?);
2709 unsafe { poller.add(&stream, Event::readable(0)) }
2711 .map_err(|e| io_err("realtime register failed", e))?;
2712 let registration = SocketRegistration {
2713 poller: Arc::clone(&poller),
2714 stream,
2715 };
2716 let (outgoing, queued) = mpsc::sync_channel::<Outgoing>(1);
2717 self.outgoing = Some(Box::new(SocketSender {
2718 queue: outgoing,
2719 poller,
2720 }));
2721 self.reader_stop = Arc::new(AtomicBool::new(false));
2722 self.round = Arc::new(RoundChannel::default());
2723 let inbound = Arc::clone(&self.inbound);
2724 let stop = Arc::clone(&self.reader_stop);
2725 let round = Arc::clone(&self.round);
2726 let silence = 2 * self
2731 .policy
2732 .realtime_heartbeat
2733 .unwrap_or(Duration::from_secs(30));
2734 self.reader = Some(std::thread::spawn(move || {
2735 let mut events = Events::new();
2736 let mut pending: Option<mpsc::Sender<Result<(), TransportError>>> = None;
2737 let mut heard: Option<Instant> = None;
2738 let failure = 'io: loop {
2739 if stop.load(Ordering::SeqCst) {
2740 let _ = ws.close(None);
2741 let _ = ws.flush();
2742 break TransportError::new(
2743 "sync.transport_failed",
2744 "realtime disconnected mid-round (§8.7)",
2745 );
2746 }
2747 if pending.is_none() {
2748 if let Ok(outgoing) = queued.try_recv() {
2749 pending = Some(outgoing.completed);
2750 if let Err(error) = ws.write(outgoing.message) {
2752 if !is_would_block(&error) {
2753 break ws_err("realtime write failed", error);
2754 }
2755 }
2756 }
2757 }
2758 let mut drained = false;
2760 for _ in 0..64 {
2761 let received = ws.read();
2762 if let Ok(message) = &received {
2763 let armed = heard.is_some()
2764 || matches!(message, Message::Text(text)
2765 if serde_json::from_str::<serde_json::Value>(text)
2766 .is_ok_and(|event| event["event"] == "heartbeat"));
2767 if armed {
2768 heard = Some(Instant::now());
2769 }
2770 let kind = match message {
2771 Message::Text(_) => Some(RealtimeReceiptKind::Text),
2772 Message::Binary(bytes) => Some(match bytes.first() {
2773 Some(&crate::REALTIME_TAG_DELTA) => RealtimeReceiptKind::Delta,
2774 Some(&crate::REALTIME_TAG_ROUND) => RealtimeReceiptKind::Round,
2775 _ => RealtimeReceiptKind::OtherBinary,
2776 }),
2777 Message::Ping(_) => Some(RealtimeReceiptKind::Ping),
2778 Message::Pong(_) => Some(RealtimeReceiptKind::Pong),
2779 Message::Close(_) => Some(RealtimeReceiptKind::Close),
2780 Message::Frame(_) => None,
2781 };
2782 if let Some(kind) = kind {
2783 let bytes = match message {
2784 Message::Close(Some(close)) => 2 + close.reason.len(),
2785 _ => message.len(),
2786 };
2787 inbound.observe(kind, bytes);
2788 }
2789 }
2790 match received {
2791 Ok(Message::Text(text)) => {
2792 inbound.push(Inbound::Text(text.to_string()))
2793 }
2794 Ok(Message::Binary(bytes)) => {
2795 if let Some(delta) = round.route_binary(&bytes) {
2796 inbound.push(Inbound::Binary(delta));
2797 }
2798 }
2799 Err(error) if is_would_block(&error) => {
2800 drained = true;
2801 break;
2802 }
2803 Ok(Message::Close(_)) => {
2804 let _ = ws.flush();
2805 break 'io TransportError::new(
2806 "sync.transport_failed",
2807 "realtime disconnected mid-round (§8.7)",
2808 );
2809 }
2810 Err(_) => {
2811 break 'io TransportError::new(
2812 "sync.transport_failed",
2813 "realtime disconnected mid-round (§8.7)",
2814 );
2815 }
2816 Ok(_) => {}
2817 }
2818 if inbound.overflowed() {
2821 let _ = ws.close(None);
2822 let _ = ws.flush();
2823 let mut error = TransportError::new(
2824 "transport.failed",
2825 "realtime inbound buffer is full",
2826 );
2827 error.details = Some(serde_json::json!({
2828 "causeKind": "inbound_overflow",
2829 "limit": super::INBOUND_BUFFER_LIMIT_BYTES,
2830 }));
2831 break 'io error;
2832 }
2833 }
2834 let interest = match ws.flush() {
2835 Ok(()) => {
2836 if let Some(completed) = pending.take() {
2837 let _ = completed.send(Ok(()));
2838 }
2839 Event::readable(0)
2840 }
2841 Err(error) if is_would_block(&error) => Event::all(0),
2842 Err(error) => break ws_err("realtime flush failed", error),
2843 };
2844 if !drained {
2845 continue;
2846 }
2847 if let Err(error) = registration.poller.modify(®istration.stream, interest) {
2848 break io_err("realtime rearm failed", error);
2849 }
2850 let wait = heard
2851 .map(|heard| (heard + silence).saturating_duration_since(Instant::now()));
2852 if wait.is_some_and(|wait| wait.is_zero()) {
2853 break timeout_error("realtime heartbeat missed", "heartbeat");
2854 }
2855 events.clear();
2856 if let Err(error) = registration.poller.wait(&mut events, wait) {
2857 if error.kind() != std::io::ErrorKind::Interrupted {
2858 break io_err("realtime wait failed", error);
2859 }
2860 }
2861 };
2862 if !stop.load(Ordering::SeqCst) {
2867 inbound.push(Inbound::Disconnected(failure.clone()));
2868 }
2869 if let Some(completed) = pending {
2870 let _ = completed.send(Err(failure.clone()));
2871 }
2872 while let Ok(outgoing) = queued.try_recv() {
2873 let _ = outgoing.completed.send(Err(failure.clone()));
2874 }
2875 round.fail_in_flight(failure);
2876 }));
2877 Ok(())
2878 }
2879
2880 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
2881 self.send_message(Message::Text(text.to_owned().into()))
2882 }
2883
2884 fn realtime_close(&mut self) -> Result<(), TransportError> {
2885 self.shutdown();
2886 Ok(())
2887 }
2888
2889 fn round_deadline(&self) -> Option<Duration> {
2890 NativeTransport::round_deadline(self)
2891 }
2892
2893 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
2894 NativeTransport::set_round_deadline(self, deadline);
2895 }
2896 }
2897 #[cfg(test)]
2898 mod transfer_tests {
2899 use super::*;
2900 use ureq::unversioned::resolver::DefaultResolver;
2901 use ureq::unversioned::transport::{ConnectionDetails, Connector};
2902
2903 const SECRET: &str = "https://user:password@cdn.example/secret-capability-path?secret-query=value#secret-fragment";
2904
2905 #[derive(Debug)]
2906 struct SecretError;
2907 impl std::fmt::Display for SecretError {
2908 fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2909 panic!("transport error must never be formatted")
2910 }
2911 }
2912 impl std::error::Error for SecretError {}
2913
2914 #[derive(Debug)]
2915 struct FailingConnector(&'static str);
2916 impl Connector for FailingConnector {
2917 type Out = ();
2918 fn connect(
2919 &self,
2920 _: &ConnectionDetails,
2921 _: Option<()>,
2922 ) -> Result<Option<()>, ureq::Error> {
2923 Err(match self.0 {
2924 "io" => ureq::Error::Io(std::io::Error::new(
2925 std::io::ErrorKind::ConnectionRefused,
2926 format!("404 {SECRET}"),
2927 )),
2928 "tls" => ureq::Error::Tls(SECRET),
2929 "request" => ureq::Error::BadUri(SECRET.into()),
2930 "timeout" => ureq::Error::Timeout(ureq::Timeout::Global),
2931 "404" => ureq::Error::StatusCode(404),
2932 "401" => ureq::Error::StatusCode(401),
2933 "403" => ureq::Error::StatusCode(403),
2934 "503" => ureq::Error::StatusCode(503),
2935 _ => ureq::Error::Other(Box::new(SecretError)),
2936 })
2937 }
2938 }
2939
2940 #[test]
2941 fn injected_http_failures_never_expose_capabilities_or_format_the_cause() {
2942 for kind in [
2943 "io", "tls", "request", "timeout", "unknown", "404", "401", "403", "503",
2944 ] {
2945 for operation in [
2946 "sync",
2947 "remote",
2948 "segment",
2949 "signed_segment",
2950 "blob",
2951 "signed_blob",
2952 "upload",
2953 "grant",
2954 "signed_put",
2955 ] {
2956 let mut transport =
2957 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None)
2958 .unwrap();
2959 transport.agent = ureq::Agent::with_parts(
2960 ureq::Agent::config_builder().build(),
2961 FailingConnector(kind),
2962 DefaultResolver::default(),
2963 );
2964 let mut progress = |_| {};
2965 let result = match operation {
2966 "sync" => transport.sync(&[]).map(|_| ()),
2967 "remote" => transport.remote_operation(&[]).map(|_| ()),
2968 "segment" => transport.download_segment(&SegmentRequest { segment_id: "sha256:test".into(), table: "tasks".into(), requested_scopes_json: "{}".into(), byte_length: 1 << 20 }, &mut progress).map(|_| ()),
2969 "signed_segment" => transport.fetch_url("http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment", 1 << 20, &mut progress).map(|_| ()),
2970 "blob" => transport.blob_download("sha256:test", None).map(|_| ()),
2971 "signed_blob" => transport.fetch_blob_url("http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment", None).map(|_| ()),
2972 "upload" => transport.blob_upload("sha256:test", &[], None),
2973 "grant" => transport.blob_upload_grant("sha256:test", 0, None).map(|_| ()),
2974 _ => transport.blob_put_url(&format!("sha256:{}", "0".repeat(64)), "http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment", &[], None),
2975 };
2976 let error = result.unwrap_err();
2977 let expected_code = match (operation, kind) {
2978 (_, "timeout") => "transport.timeout",
2979 ("segment" | "signed_segment", _) => "sync.transport_failed",
2980 ("blob", "404") => "blob.not_found",
2981 ("blob", "401") => "sync.auth_required",
2982 ("blob", "403") => "blob.forbidden",
2983 _ => "transport.failed",
2984 };
2985 assert_eq!(error.code, expected_code, "{operation} {kind}");
2986 let details = error.details.unwrap();
2987 assert_eq!(
2988 details["causeKind"],
2989 match kind {
2990 "io" => "connect",
2991 "404" | "401" | "403" | "503" => "status",
2992 other => other,
2993 }
2994 );
2995 assert_eq!(
2996 details.get("httpStatus").and_then(|value| value.as_u64()),
2997 kind.parse::<u64>().ok()
2998 );
2999 let rendered = format!("{}{}", error.message, details);
3000 for secret in [
3001 "http://",
3002 "https://",
3003 "password",
3004 "secret-capability-path",
3005 "secret-query",
3006 "secret-fragment",
3007 ] {
3008 assert!(!rendered.contains(secret), "{operation} {kind}");
3009 }
3010 assert!(details.get("causeMessage").is_none());
3011 assert!(details.get("path").is_none());
3012 }
3013 }
3014 }
3015
3016 #[test]
3017 fn injected_body_and_websocket_errors_do_not_expose_payloads_or_source_chains() {
3018 struct FailingRead;
3019 impl std::io::Read for FailingRead {
3020 fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
3021 Err(std::io::Error::other(SECRET))
3022 }
3023 }
3024 let response = ureq::http::Response::builder()
3025 .status(200)
3026 .body(ureq::Body::builder().reader(FailingRead))
3027 .unwrap();
3028 let error = segment_error(read_body(response, None, None, None).unwrap_err());
3029 assert_eq!(error.code, "sync.transport_failed");
3030 assert_eq!(error.message, "segment transfer failed");
3031 assert_eq!(
3032 error.details,
3033 Some(serde_json::json!({ "causeKind": "body", "httpStatus": 200 }))
3034 );
3035 for cause in [
3036 tungstenite::Error::Io(std::io::Error::other(SecretError)),
3037 tungstenite::Error::Http(Box::new(
3038 tungstenite::http::Response::builder()
3039 .status(403)
3040 .header("location", SECRET)
3041 .body(Some(SECRET.as_bytes().to_vec()))
3042 .unwrap(),
3043 )),
3044 ] {
3045 let error = ws_err("realtime connection failed", cause);
3046 assert_eq!(error.message, "realtime connection failed");
3047 let details = error.details.unwrap();
3048 assert!(details.get("path").is_none());
3049 assert!(details.get("causeMessage").is_none());
3050 assert!(!details.to_string().contains("secret"));
3051 }
3052 }
3053
3054 #[test]
3055 fn body_read_timeouts_are_classified_through_ureq_wrapping() {
3056 let direct = ureq::Error::Timeout(ureq::Timeout::Global).into_io();
3058 assert!(is_timeout_io(&direct));
3059 let wrapped = ureq::Error::Decompress(
3061 "gzip",
3062 ureq::Error::Timeout(ureq::Timeout::Global).into_io(),
3063 )
3064 .into_io();
3065 assert!(is_timeout_io(&wrapped));
3066 assert!(is_timeout_io(&std::io::Error::new(
3068 std::io::ErrorKind::TimedOut,
3069 "plain"
3070 )));
3071 assert!(!is_timeout_io(&std::io::Error::other("other")));
3072 assert!(!is_timeout_io(&ureq::Error::StatusCode(500).into_io()));
3073 }
3074
3075 #[test]
3076 fn body_read_wrapped_timeouts_surface_the_typed_code() {
3077 struct TimeoutRead(bool);
3078 impl std::io::Read for TimeoutRead {
3079 fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
3080 Err(if self.0 {
3081 ureq::Error::Timeout(ureq::Timeout::Global).into_io()
3082 } else {
3083 ureq::Error::Decompress(
3084 "gzip",
3085 ureq::Error::Timeout(ureq::Timeout::Global).into_io(),
3086 )
3087 .into_io()
3088 })
3089 }
3090 }
3091 for compressed in [false, true] {
3092 let response = ureq::http::Response::builder()
3093 .status(200)
3094 .body(ureq::Body::builder().reader(TimeoutRead(compressed)))
3095 .unwrap();
3096 let error = read_body(response, None, None, None).unwrap_err();
3097 assert_eq!(error.code, "transport.timeout", "{compressed}");
3098 assert_eq!(
3099 error.details,
3100 Some(serde_json::json!({ "causeKind": "timeout" }))
3101 );
3102 }
3103 }
3104
3105 #[test]
3106 fn policy_validation_rejects_zero_and_unrepresentable_bounds() {
3107 assert_eq!(
3108 HostTransportPolicy {
3109 request_timeout: Some(Duration::ZERO),
3110 ..Default::default()
3111 }
3112 .validate()
3113 .unwrap_err(),
3114 "sync.invalid_request: transport config requestTimeoutMs is out of range"
3115 );
3116 assert_eq!(
3117 HostTransportPolicy {
3118 round_deadline: Some(Duration::MAX),
3119 ..Default::default()
3120 }
3121 .validate()
3122 .unwrap_err(),
3123 "sync.invalid_request: transport config roundDeadlineMs is out of range"
3124 );
3125 for interval in [Duration::ZERO, Duration::MAX / 2 + Duration::from_secs(1)] {
3126 assert_eq!(
3127 HostTransportPolicy {
3128 realtime_heartbeat: Some(interval),
3129 ..Default::default()
3130 }
3131 .validate()
3132 .unwrap_err(),
3133 "sync.invalid_request: transport config realtimeHeartbeatMs is out of range"
3134 );
3135 }
3136 assert!(HostTransportPolicy {
3137 max_request_bytes: Some(0),
3138 ..Default::default()
3139 }
3140 .validate()
3141 .is_err());
3142 assert!(HostTransportPolicy {
3143 max_response_bytes: Some(0),
3144 ..Default::default()
3145 }
3146 .validate()
3147 .is_err());
3148 assert!(HostTransportPolicy::default().validate().is_ok());
3151 assert_eq!(
3152 HostTransportPolicy::from_config(&serde_json::json!({ "requestTimeoutMs": 0 }))
3153 .unwrap_err(),
3154 "sync.invalid_request: transport config requestTimeoutMs must be a positive integer"
3155 );
3156 }
3157
3158 #[test]
3159 fn set_policy_rejects_invalid_and_preserves_the_current_policy() {
3160 let mut transport =
3161 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None).unwrap();
3162 assert_eq!(transport.policy.max_response_bytes, None);
3163 let error = transport
3164 .set_policy(HostTransportPolicy {
3165 max_response_bytes: Some(0),
3166 ..Default::default()
3167 })
3168 .unwrap_err();
3169 assert_eq!(
3170 error,
3171 "sync.invalid_request: transport config maxResponseBytes must be a positive integer"
3172 );
3173 assert_eq!(transport.policy.max_response_bytes, None);
3174 }
3175
3176 #[test]
3177 fn an_expired_round_deadline_is_typed_and_cleared_for_the_next_round() {
3178 let mut transport =
3179 NativeTransport::new("http://127.0.0.1:9", &serde_json::json!({}), None).unwrap();
3180 transport.set_round_deadline(Some(std::time::Instant::now() - Duration::from_secs(1)));
3181 let error = transport.blob_download("sha256:test", None).unwrap_err();
3182 assert_eq!(error.code, "transport.timeout");
3183 assert_eq!(
3184 error.details,
3185 Some(serde_json::json!({ "causeKind": "round_deadline" }))
3186 );
3187 transport.set_round_deadline(None);
3189 assert!(transport.round_remaining().is_ok());
3190 }
3191
3192 #[test]
3193 fn an_expired_round_deadline_refuses_before_queuing_a_realtime_send() {
3194 let mut transport =
3195 NativeTransport::new("http://127.0.0.1:9", &serde_json::json!({}), None).unwrap();
3196 transport.set_round_deadline(Some(std::time::Instant::now() - Duration::from_secs(1)));
3197 let error = transport.realtime_send("blocked").unwrap_err();
3201 assert_eq!(error.code, "transport.timeout");
3202 assert_eq!(
3203 error.details,
3204 Some(serde_json::json!({ "causeKind": "round_deadline" }))
3205 );
3206 transport.set_round_deadline(None);
3209 let error = transport.realtime_send("blocked").unwrap_err();
3210 assert_eq!(error.code, "transport.failed");
3211 }
3212
3213 #[test]
3214 fn the_request_timeout_bounds_phases_and_the_round_deadline_bounds_the_call() {
3215 #[derive(Debug)]
3216 struct TimeoutRecording(Arc<Mutex<Vec<Option<Duration>>>>);
3217 impl Connector for TimeoutRecording {
3218 type Out = ();
3219 fn connect(
3220 &self,
3221 details: &ConnectionDetails,
3222 _: Option<()>,
3223 ) -> Result<Option<()>, ureq::Error> {
3224 self.0
3225 .lock()
3226 .unwrap()
3227 .push(details.config.timeouts().global);
3228 Err(ureq::Error::ConnectionFailed)
3229 }
3230 }
3231 let mut transport =
3232 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None).unwrap();
3233 transport
3234 .set_policy(HostTransportPolicy {
3235 request_timeout: Some(Duration::from_millis(1234)),
3236 ..Default::default()
3237 })
3238 .unwrap();
3239 let timeouts = transport.agent.config().timeouts();
3240 for phase in [
3241 timeouts.resolve,
3242 timeouts.connect,
3243 timeouts.send_request,
3244 timeouts.recv_response,
3245 ] {
3246 assert_eq!(phase, Some(Duration::from_millis(1234)));
3247 }
3248 assert_eq!(timeouts.global, None, "the request bound is not end to end");
3249 assert_eq!(timeouts.recv_body, None, "a body is bounded by silence");
3250 let seen = Arc::new(Mutex::new(Vec::new()));
3251 transport.agent = ureq::Agent::with_parts(
3252 ureq::Agent::config_builder().build(),
3253 TimeoutRecording(Arc::clone(&seen)),
3254 DefaultResolver::default(),
3255 );
3256 let _ = transport.blob_upload("sha256:test", &[0u8; 4], None);
3257 transport.set_round_deadline(Some(std::time::Instant::now() + Duration::from_secs(60)));
3258 let _ = transport.blob_upload("sha256:test", &[0u8; 4], None);
3259 let seen = seen.lock().unwrap();
3260 assert_eq!(seen[0], None, "no round deadline, no call ceiling");
3261 assert!(seen[1].is_some_and(|ceiling| ceiling <= Duration::from_secs(60)));
3262 }
3263
3264 #[test]
3265 fn the_idle_bound_replaces_an_unbounded_or_longer_wait_and_keeps_a_shorter_one() {
3266 let idle = Duration::from_secs(30);
3267 let unbounded = NextTimeout::default();
3268 assert_eq!(idle_bound(unbounded, idle).after, UreqDuration::Exact(idle));
3269 let longer = NextTimeout {
3270 after: UreqDuration::from_secs(90),
3271 per_read: Some(UreqDuration::from_secs(7)),
3272 ..NextTimeout::default()
3273 };
3274 let bounded = idle_bound(longer, idle);
3275 assert_eq!(bounded.after, UreqDuration::Exact(idle));
3276 assert_eq!(bounded.reason, UreqTimeout::RecvBody);
3277 assert_eq!(
3278 bounded.per_read, longer.per_read,
3279 "per-read cap passes through"
3280 );
3281 let shorter = NextTimeout {
3282 after: UreqDuration::from_secs(5),
3283 ..NextTimeout::default()
3284 };
3285 assert_eq!(idle_bound(shorter, idle), shorter);
3286 }
3287
3288 fn shaped_link(
3293 total: u64,
3294 chunk: u64,
3295 latency: Duration,
3296 rate: u64,
3297 stalls: &[(Duration, Duration)],
3298 ) -> Vec<Duration> {
3299 let mut at = latency;
3300 let mut arrivals = Vec::new();
3301 let mut left = total;
3302 while left > 0 {
3303 let piece = left.min(chunk);
3304 at += Duration::from_micros(piece * 1_000_000 / rate);
3305 for (start, length) in stalls {
3306 if at > *start && at < *start + *length {
3307 at = *start + *length;
3308 }
3309 }
3310 arrivals.push(at);
3311 left -= piece;
3312 }
3313 arrivals
3314 }
3315
3316 fn round_over(
3320 arrivals: &[Duration],
3321 idle: Duration,
3322 ceiling: Option<Duration>,
3323 ) -> Result<Duration, (Duration, bool)> {
3324 let base = Instant::now();
3325 let mut last = base;
3326 for arrival in arrivals {
3327 let (deadline, ceiling_binds) =
3328 round_expiry(last, idle, ceiling.map(|ceiling| base + ceiling));
3329 if base + *arrival > deadline {
3330 return Err((deadline - base, ceiling_binds));
3331 }
3332 last = base + *arrival;
3333 }
3334 Ok(last - base)
3335 }
3336
3337 #[test]
3338 fn a_slow_link_that_keeps_delivering_never_trips_the_idle_bound() {
3339 let link = shaped_link(
3341 8_600_000,
3342 64 * 1024,
3343 Duration::from_millis(80),
3344 100_000,
3345 &[],
3346 );
3347 let finished = round_over(&link, ROUND_IDLE_TIMEOUT, None).unwrap();
3348 assert!(
3349 finished > ROUND_IDLE_TIMEOUT * 2,
3350 "the transfer outlasts the old whole-transfer bound ({finished:?})"
3351 );
3352 }
3353
3354 #[test]
3355 fn a_stall_shorter_than_the_idle_bound_is_survived_and_a_longer_one_is_not() {
3356 let link = |length| {
3357 shaped_link(
3358 4_000_000,
3359 64 * 1024,
3360 Duration::from_millis(80),
3361 100_000,
3362 &[(Duration::from_secs(10), length)],
3363 )
3364 };
3365 round_over(&link(Duration::from_secs(29)), ROUND_IDLE_TIMEOUT, None).unwrap();
3366 let (expired, ceiling_bound) =
3367 round_over(&link(Duration::from_secs(31)), ROUND_IDLE_TIMEOUT, None).unwrap_err();
3368 assert!(!ceiling_bound);
3369 assert!(
3370 expired >= Duration::from_secs(30) && expired < Duration::from_secs(41),
3371 "the bound counts from the last chunk before the stall ({expired:?})"
3372 );
3373 }
3374
3375 #[test]
3376 fn the_host_ceiling_still_bounds_a_link_that_keeps_delivering() {
3377 let link = shaped_link(
3378 8_600_000,
3379 64 * 1024,
3380 Duration::from_millis(80),
3381 100_000,
3382 &[],
3383 );
3384 let (expired, ceiling_bound) =
3385 round_over(&link, ROUND_IDLE_TIMEOUT, Some(Duration::from_secs(60))).unwrap_err();
3386 assert!(ceiling_bound);
3387 assert_eq!(expired, Duration::from_secs(60));
3388 }
3389
3390 #[test]
3391 fn an_expired_round_deadline_bounds_the_socket_wait() {
3392 let channel = RoundChannel::default();
3393 let error = channel
3394 .wait(
3395 Some(std::time::Instant::now() - Duration::from_secs(1)),
3396 ROUND_IDLE_TIMEOUT,
3397 )
3398 .unwrap_err();
3399 assert_eq!(error.code, "transport.timeout");
3400 assert_eq!(
3401 error.details,
3402 Some(serde_json::json!({ "causeKind": "round_deadline" }))
3403 );
3404 }
3405 }
3406}
3407
3408#[cfg(all(test, feature = "native-transport"))]
3409mod tests {
3410 use super::*;
3411 use std::io::{BufRead, BufReader, Write};
3412
3413 #[test]
3414 fn an_undrained_inbound_buffer_past_its_limit_ends_the_connection() {
3415 const DELTA_BYTES: usize = 1024 * 1024;
3416 let deltas = INBOUND_BUFFER_LIMIT_BYTES / DELTA_BYTES + 1;
3417 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3418 let base = format!("http://{}", listener.local_addr().unwrap());
3419 let server = std::thread::spawn(move || {
3420 let (socket, _) = listener.accept().unwrap();
3421 let mut socket = tungstenite::accept(socket).unwrap();
3422 for _ in 0..deltas {
3423 let mut frame = vec![crate::REALTIME_TAG_DELTA];
3424 frame.resize(DELTA_BYTES + 1, 7);
3425 socket
3426 .send(tungstenite::Message::Binary(frame.into()))
3427 .unwrap();
3428 }
3429 while !matches!(socket.read(), Ok(tungstenite::Message::Close(_)) | Err(_)) {}
3430 });
3431 let (woke, wakes) = mpsc::channel::<()>();
3432 let mut transport = HostTransport::from_config_with_notify(
3433 &serde_json::json!({"baseUrl": base}),
3434 Some(Arc::new(move || {
3435 let _ = woke.send(());
3436 })),
3437 )
3438 .unwrap();
3439 transport.realtime_connect().unwrap();
3440 for _ in 0..=deltas {
3442 wakes.recv().unwrap();
3443 }
3444 let inbound = transport.take_inbound();
3445 assert_eq!(inbound.len(), deltas + 1);
3446 assert!(inbound[..deltas]
3447 .iter()
3448 .all(|frame| matches!(frame, Inbound::Binary(bytes) if bytes.len() == DELTA_BYTES)));
3449 assert!(matches!(
3450 &inbound[deltas],
3451 Inbound::Disconnected(error) if error.code == "transport.failed"
3452 && error.details.as_ref().and_then(|d| d.get("causeKind"))
3453 == Some(&serde_json::json!("inbound_overflow"))
3454 ));
3455 transport.shutdown();
3456 server.join().unwrap();
3457 }
3458
3459 #[test]
3460 fn the_first_heartbeat_arms_silence_detection() {
3461 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3462 let base = format!("http://{}", listener.local_addr().unwrap());
3463 let (beat, heartbeat) = mpsc::channel::<()>();
3464 let server = std::thread::spawn(move || {
3465 let (socket, _) = listener.accept().unwrap();
3466 let mut socket = tungstenite::accept(socket).unwrap();
3467 socket
3468 .send(tungstenite::Message::Text("{\"event\":\"hello\"}".into()))
3469 .unwrap();
3470 heartbeat.recv().unwrap();
3471 socket
3472 .send(tungstenite::Message::Text(
3473 "{\"event\":\"heartbeat\",\"data\":{\"timestamp\":1}}".into(),
3474 ))
3475 .unwrap();
3476 while socket.read().is_ok() {}
3478 });
3479 let (woke, wakes) = mpsc::channel::<()>();
3480 let mut transport = HostTransport::from_config_with_notify(
3481 &serde_json::json!({"baseUrl": base, "realtimeHeartbeatMs": 50}),
3482 Some(Arc::new(move || {
3483 let _ = woke.send(());
3484 })),
3485 )
3486 .unwrap();
3487 transport.realtime_connect().unwrap();
3488 wakes.recv_timeout(Duration::from_secs(10)).unwrap();
3489 assert!(wakes.recv_timeout(Duration::from_millis(250)).is_err());
3492 beat.send(()).unwrap();
3493 for _ in 0..2 {
3494 wakes.recv_timeout(Duration::from_secs(10)).unwrap();
3495 }
3496 let inbound = transport.take_inbound();
3497 assert_eq!(inbound.len(), 3);
3498 assert!(matches!(&inbound[0], Inbound::Text(_)));
3499 assert!(matches!(&inbound[1], Inbound::Text(text) if text.contains("heartbeat")));
3500 assert!(matches!(
3501 &inbound[2],
3502 Inbound::Disconnected(error) if error.code == "transport.timeout"
3503 && error.details == Some(serde_json::json!({ "causeKind": "heartbeat" }))
3504 ));
3505 transport.shutdown();
3506 server.join().unwrap();
3507 }
3508
3509 #[test]
3510 fn segment_and_blob_reads_stop_past_the_declared_byte_length() {
3511 for case in ["segment", "signed_segment", "blob", "signed_blob"] {
3512 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3513 let base = format!("http://{}", listener.local_addr().unwrap());
3514 let server = std::thread::spawn(move || {
3515 let (mut socket, _) = listener.accept().unwrap();
3516 let mut reader = BufReader::new(socket.try_clone().unwrap());
3517 loop {
3518 let mut line = String::new();
3519 reader.read_line(&mut line).unwrap();
3520 if line == "\r\n" {
3521 break;
3522 }
3523 }
3524 let body = vec![7u8; 4 << 20];
3525 let _ = write!(
3527 socket,
3528 "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
3529 body.len()
3530 );
3531 let _ = socket.write_all(&body);
3532 });
3533 let mut transport =
3534 HostTransport::from_config(&serde_json::json!({ "baseUrl": base })).unwrap();
3535 let mut progress = |_| {};
3536 let bytes = match case {
3537 "segment" => transport.download_segment(
3538 &SegmentRequest {
3539 segment_id: "sha256:test".into(),
3540 table: "tasks".into(),
3541 requested_scopes_json: "{}".into(),
3542 byte_length: 16,
3543 },
3544 &mut progress,
3545 ),
3546 "signed_segment" => {
3547 transport.fetch_url(&format!("{base}/signed"), 16, &mut progress)
3548 }
3549 "blob" => transport
3550 .blob_download("sha256:test", Some(16))
3551 .map(|download| match download {
3552 BlobDownload::Bytes(bytes) => bytes,
3553 BlobDownload::Url { .. } => panic!("expected inline bytes"),
3554 }),
3555 _ => transport.fetch_blob_url(&format!("{base}/signed"), Some(16)),
3556 }
3557 .unwrap();
3558 assert!(
3561 bytes.len() > 16 && bytes.len() <= 16 + 64 * 1024,
3562 "{case} read {} bytes",
3563 bytes.len()
3564 );
3565 server.join().unwrap();
3566 }
3567 }
3568
3569 #[test]
3570 fn a_round_deadline_expiring_during_a_socket_send_is_the_typed_timeout() {
3571 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3572 let base = format!("http://{}", listener.local_addr().unwrap());
3573 let (release, released) = mpsc::channel::<()>();
3574 let server = std::thread::spawn(move || {
3577 let (socket, _) = listener.accept().unwrap();
3578 let _socket = tungstenite::accept(socket).unwrap();
3579 let _ = released.recv();
3580 });
3581 let mut transport =
3582 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3583 transport.realtime_connect().unwrap();
3584 transport.set_round_deadline(Some(std::time::Instant::now() + Duration::from_millis(200)));
3585 let error = transport
3586 .realtime_sync(&vec![0u8; 32 * 1024 * 1024])
3587 .unwrap_err();
3588 assert_eq!(error.code, "transport.timeout");
3589 assert_eq!(
3590 error.details,
3591 Some(serde_json::json!({ "causeKind": "round_deadline" }))
3592 );
3593 assert!(matches!(
3594 &transport.take_inbound()[..],
3595 [Inbound::Disconnected(lost)] if lost.code == "transport.timeout"
3596 ));
3597 transport.shutdown();
3598 release.send(()).unwrap();
3599 server.join().unwrap();
3600 }
3601
3602 #[test]
3603 fn receipt_observer_covers_round_chunks_wakes_reconnect_and_overflow() {
3604 use ssp2::model::{Frame, Message, MsgKind};
3605 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3606 let base = format!("http://{}", listener.local_addr().unwrap());
3607 let response = ssp2::encode_message(&Message {
3608 wire_version: ssp2::decode::WIRE_VERSION,
3609 msg_kind: MsgKind::Response,
3610 frames: vec![Frame::RespHeader {
3611 required_schema_version: None,
3612 latest_schema_version: None,
3613 log_epoch: "epoch".into(),
3614 reset_required: false,
3615 }],
3616 });
3617 let server_response = response.clone();
3618 let server = std::thread::spawn(move || {
3619 for _ in 0..2 {
3620 let (socket, _) = listener.accept().unwrap();
3621 let mut socket = tungstenite::accept(socket).unwrap();
3622 assert!(matches!(
3623 socket.read().unwrap(),
3624 tungstenite::Message::Binary(_)
3625 ));
3626 socket
3627 .send(tungstenite::Message::Binary(
3628 vec![crate::REALTIME_TAG_DELTA, 7].into(),
3629 ))
3630 .unwrap();
3631 socket
3632 .send(tungstenite::Message::Text("wake".into()))
3633 .unwrap();
3634 let split = server_response.len() / 2;
3635 for part in [&server_response[..split], &server_response[split..]] {
3636 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
3637 chunk.extend_from_slice(part);
3638 socket
3639 .send(tungstenite::Message::Binary(chunk.into()))
3640 .unwrap();
3641 }
3642 assert!(matches!(
3643 socket.read().unwrap(),
3644 tungstenite::Message::Close(_)
3645 ));
3646 }
3647 });
3648 let mut transport =
3649 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3650 let (sender, receiver) = mpsc::sync_channel(4);
3651 let (small_sender, small_receiver) = mpsc::sync_channel(1);
3652 transport.set_realtime_receipt_observer(Some(sender));
3653 for round in 0..2 {
3654 transport.realtime_connect().unwrap();
3655 let before = SystemTime::now();
3656 assert_eq!(transport.realtime_sync(b"request").unwrap(), response);
3657 let after = SystemTime::now();
3658 if round == 0 {
3659 let receipts = [
3660 receiver.recv().unwrap(),
3661 receiver.recv().unwrap(),
3662 receiver.recv().unwrap(),
3663 receiver.recv().unwrap(),
3664 ];
3665 assert_eq!(
3666 receipts.map(|r| r.kind),
3667 [
3668 RealtimeReceiptKind::Delta,
3669 RealtimeReceiptKind::Text,
3670 RealtimeReceiptKind::Round,
3671 RealtimeReceiptKind::Round
3672 ]
3673 );
3674 assert_eq!(
3675 receipts.map(|r| r.bytes),
3676 [
3677 2,
3678 4,
3679 1 + response.len() / 2,
3680 1 + response.len() - response.len() / 2
3681 ]
3682 );
3683 assert!(receipts
3684 .iter()
3685 .all(|r| r.received_at >= before && r.received_at <= after));
3686 assert!(
3687 matches!(&transport.take_inbound()[..], [Inbound::Binary(bytes), Inbound::Text(text)] if bytes == &[7] && text == "wake")
3688 );
3689 assert_eq!(transport.take_dropped_realtime_receipts(), 0);
3690 transport.set_realtime_receipt_observer(Some(small_sender.clone()));
3693 } else {
3694 assert_eq!(
3695 small_receiver.recv().unwrap().kind,
3696 RealtimeReceiptKind::Delta
3697 );
3698 assert_eq!(transport.take_dropped_realtime_receipts(), 3);
3699 assert_eq!(transport.take_dropped_realtime_receipts(), 0);
3700 }
3701 transport.shutdown();
3702 }
3703 server.join().unwrap();
3704 }
3705
3706 #[test]
3707 fn receipt_observer_reports_chunks_of_a_failed_round() {
3708 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3709 let base = format!("http://{}", listener.local_addr().unwrap());
3710 let server = std::thread::spawn(move || {
3711 let (socket, _) = listener.accept().unwrap();
3712 let mut socket = tungstenite::accept(socket).unwrap();
3713 assert!(matches!(
3714 socket.read().unwrap(),
3715 tungstenite::Message::Binary(_)
3716 ));
3717 socket
3718 .send(tungstenite::Message::Binary(
3719 vec![crate::REALTIME_TAG_ROUND, b'S'].into(),
3720 ))
3721 .unwrap();
3722 socket
3723 .close(Some(tungstenite::protocol::CloseFrame {
3724 code: tungstenite::protocol::frame::coding::CloseCode::Normal,
3725 reason: "bye".into(),
3726 }))
3727 .unwrap();
3728 });
3729 let mut transport =
3730 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3731 let (sender, receiver) = mpsc::sync_channel(2);
3732 transport.set_realtime_receipt_observer(Some(sender));
3733 transport.realtime_connect().unwrap();
3734 assert!(transport.realtime_sync(b"request").is_err());
3735 assert_eq!(receiver.recv().unwrap().kind, RealtimeReceiptKind::Round);
3736 let close = receiver.recv().unwrap();
3737 assert_eq!(close.kind, RealtimeReceiptKind::Close);
3738 assert_eq!(close.bytes, 5);
3739 assert_eq!(transport.take_dropped_realtime_receipts(), 0);
3740 transport.shutdown();
3741 server.join().unwrap();
3742 }
3743
3744 #[test]
3745 fn detached_round_uses_the_owned_socket_and_shutdown_releases_it() {
3746 use ssp2::model::{Frame, Message, MsgKind};
3747 use std::sync::mpsc::channel;
3748 for cancelled in [false, true] {
3749 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3750 let base = format!("http://{}", listener.local_addr().unwrap());
3751 let (entered_tx, entered_rx) = channel();
3752 let (release_tx, release_rx) = channel();
3753 let server = std::thread::spawn(move || {
3754 let (socket, _) = listener.accept().unwrap();
3755 let mut socket = tungstenite::accept(socket).unwrap();
3756 let request = socket.read().unwrap().into_data();
3757 assert_eq!(request[0], crate::REALTIME_TAG_ROUND);
3758 assert_eq!(
3759 ssp2::decode_message(&request[1..]).unwrap().msg_kind,
3760 MsgKind::Request
3761 );
3762 entered_tx.send(()).unwrap();
3763 release_rx.recv().unwrap();
3764 if !cancelled {
3765 let response = ssp2::encode_message(&Message {
3766 wire_version: ssp2::decode::WIRE_VERSION,
3767 msg_kind: MsgKind::Response,
3768 frames: vec![Frame::RespHeader {
3769 required_schema_version: None,
3770 latest_schema_version: None,
3771 log_epoch: "epoch".into(),
3772 reset_required: true,
3773 }],
3774 });
3775 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
3776 chunk.extend(response);
3777 socket
3778 .send(tungstenite::Message::Binary(chunk.into()))
3779 .unwrap();
3780 }
3781 });
3782 let mut owner =
3783 HostTransport::new_from_config(&serde_json::json!({"baseUrl":base})).unwrap();
3784 owner.realtime_connect_for_client("detached-round").unwrap();
3785 let mut detached = owner.fork_round();
3786 let network = std::thread::spawn(move || {
3787 let request = ssp2::encode_message(&Message {
3788 wire_version: ssp2::decode::WIRE_VERSION,
3789 msg_kind: MsgKind::Request,
3790 frames: vec![
3791 Frame::ReqHeader {
3792 client_id: "detached-round".into(),
3793 schema_version: 1,
3794 log_epoch: None,
3795 outbox_id: None,
3796 },
3797 Frame::PullHeader {
3798 limit_commits: 0,
3799 limit_snapshot_rows: 0,
3800 max_snapshot_pages: 0,
3801 accept: 3,
3802 cursor: -1,
3803 registration_digest: ssp2::registration::registration_digest([]),
3804 echo_digest: None,
3805 },
3806 ],
3807 });
3808 detached.realtime_sync(&request)
3809 });
3810 entered_rx.recv().unwrap();
3811 owner.set_headers(vec![("authorization".into(), "Bearer rotated".into())]);
3812 assert!(owner.take_inbound().is_empty());
3813 if cancelled {
3814 owner.shutdown();
3815 }
3816 release_tx.send(()).unwrap();
3817 let response = network.join().unwrap();
3818 if cancelled {
3819 assert!(response.is_err());
3820 } else {
3821 assert_eq!(
3822 ssp2::decode_message(&response.unwrap()).unwrap().msg_kind,
3823 MsgKind::Response
3824 );
3825 owner.shutdown();
3826 }
3827 server.join().unwrap();
3828 }
3829 }
3830
3831 fn minimal_response() -> Vec<u8> {
3832 use ssp2::model::{Frame, Message, MsgKind};
3833 ssp2::encode_message(&Message {
3834 wire_version: ssp2::decode::WIRE_VERSION,
3835 msg_kind: MsgKind::Response,
3836 frames: vec![Frame::RespHeader {
3837 required_schema_version: None,
3838 latest_schema_version: None,
3839 log_epoch: "epoch".into(),
3840 reset_required: true,
3841 }],
3842 })
3843 }
3844
3845 #[test]
3846 fn a_presigned_blob_put_sends_the_blob_id_digest_without_hashing_the_body() {
3847 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3848 let base = format!("http://{}", listener.local_addr().unwrap());
3849 let server = std::thread::spawn(move || {
3850 let (mut socket, _) = listener.accept().unwrap();
3851 let mut reader = BufReader::new(socket.try_clone().unwrap());
3852 let mut head = Vec::new();
3853 loop {
3854 let mut line = String::new();
3855 reader.read_line(&mut line).unwrap();
3856 if line == "\r\n" {
3857 break;
3858 }
3859 let (name, value) = line.trim_end().split_once(": ").unwrap_or((&line, ""));
3860 head.push(format!("{}: {value}", name.to_ascii_lowercase()));
3861 }
3862 socket
3863 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
3864 .unwrap();
3865 head
3866 });
3867 let mut transport =
3868 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3869 let url = format!("{base}/bucket/key?X-Amz-Signature=x");
3870 let error = transport
3872 .blob_put_url("sha256:AB", &url, b"body", None)
3873 .unwrap_err();
3874 assert_eq!(error.code, "sync.invalid_request");
3875 transport
3878 .blob_put_url(&format!("sha256:{}", "ab".repeat(32)), &url, b"body", None)
3879 .unwrap();
3880 let head = server.join().unwrap();
3881 assert!(head.contains(
3882 &"x-amz-checksum-sha256: q6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6s=".to_owned()
3883 ));
3884 assert!(head.contains(&"content-length: 4".to_owned()));
3885 }
3886
3887 #[test]
3888 fn a_stalled_response_body_times_out_on_silence() {
3889 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3890 let base = format!("http://{}", listener.local_addr().unwrap());
3891 let server = std::thread::spawn(move || {
3892 let (mut socket, _) = listener.accept().unwrap();
3893 let mut reader = BufReader::new(socket.try_clone().unwrap());
3894 loop {
3895 let mut line = String::new();
3896 reader.read_line(&mut line).unwrap();
3897 if line == "\r\n" {
3898 break;
3899 }
3900 }
3901 socket
3902 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n0123456789")
3903 .unwrap();
3904 let mut rest = Vec::new();
3906 let _ = std::io::Read::read_to_end(&mut reader, &mut rest);
3907 });
3908 let mut transport = HostTransport::new_from_config(
3909 &serde_json::json!({"baseUrl": base, "requestTimeoutMs": 200}),
3910 )
3911 .unwrap();
3912 let mut received = 0;
3913 let error = transport
3914 .download_segment(
3915 &SegmentRequest {
3916 segment_id: "sha256:test".into(),
3917 table: "tasks".into(),
3918 requested_scopes_json: "{}".into(),
3919 byte_length: 1 << 20,
3920 },
3921 &mut |bytes| received = bytes,
3922 )
3923 .unwrap_err();
3924 assert_eq!(error.code, "transport.timeout");
3925 assert_eq!(
3926 error.details,
3927 Some(serde_json::json!({ "causeKind": "timeout" }))
3928 );
3929 server.join().unwrap();
3930 }
3931
3932 #[test]
3933 fn a_timed_out_socket_round_closes_the_socket_and_reports_the_loss() {
3934 use std::sync::mpsc::channel;
3935 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3936 let base = format!("http://{}", listener.local_addr().unwrap());
3937 let (closed_tx, closed_rx) = channel();
3938 let server = std::thread::spawn(move || {
3939 let (socket, _) = listener.accept().unwrap();
3940 let mut socket = tungstenite::accept(socket).unwrap();
3941 socket.read().unwrap();
3942 let response = minimal_response();
3944 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
3945 chunk.extend_from_slice(&response[..response.len() / 2]);
3946 socket
3947 .send(tungstenite::Message::Binary(chunk.into()))
3948 .unwrap();
3949 while let Ok(message) = socket.read() {
3951 if matches!(message, tungstenite::Message::Close(_)) {
3952 break;
3953 }
3954 }
3955 closed_tx.send(()).unwrap();
3956 });
3957 let mut transport =
3958 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3959 transport.realtime_connect_for_client("slow").unwrap();
3960 transport.set_round_deadline(Some(
3961 std::time::Instant::now() + std::time::Duration::from_millis(300),
3962 ));
3963 let error = transport.realtime_sync(b"request").unwrap_err();
3964 assert_eq!(error.code, "transport.timeout");
3965 closed_rx.recv().unwrap();
3966 let lost = transport
3967 .take_inbound()
3968 .into_iter()
3969 .filter(|inbound| matches!(inbound, Inbound::Disconnected(e) if e.code == "transport.timeout"))
3970 .count();
3971 assert_eq!(lost, 1, "the loss is reported once");
3972 server.join().unwrap();
3973 }
3974
3975 #[test]
3976 fn a_request_larger_than_a_chunk_rides_as_several_round_chunks() {
3977 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3978 let base = format!("http://{}", listener.local_addr().unwrap());
3979 let request: Vec<u8> = (0..3 * 64 * 1024 + 5).map(|i| (i % 251) as u8).collect();
3980 let expected = request.clone();
3981 let server = std::thread::spawn(move || {
3982 let (socket, _) = listener.accept().unwrap();
3983 let mut socket = tungstenite::accept(socket).unwrap();
3984 let mut body = Vec::new();
3985 let mut messages = 0;
3986 while body.len() < expected.len() {
3987 let message = socket.read().unwrap().into_data();
3988 assert_eq!(message[0], crate::REALTIME_TAG_ROUND);
3989 body.extend_from_slice(&message[1..]);
3990 messages += 1;
3991 }
3992 assert_eq!(body, expected);
3993 assert_eq!(messages, 4);
3994 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
3995 chunk.extend(minimal_response());
3996 socket
3997 .send(tungstenite::Message::Binary(chunk.into()))
3998 .unwrap();
3999 });
4000 let mut transport =
4001 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
4002 transport.realtime_connect_for_client("chunked").unwrap();
4003 let response = transport.realtime_sync(&request).unwrap();
4004 assert_eq!(response, minimal_response());
4005 transport.shutdown();
4006 server.join().unwrap();
4007 }
4008
4009 #[test]
4010 fn segment_http_failures_have_named_codes_and_safe_cause_details() {
4011 for signed in [false, true] {
4012 for body_failure in [false, true] {
4013 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4014 let base = format!("http://{}", listener.local_addr().unwrap());
4015 let server = std::thread::spawn(move || {
4016 let (mut socket, _) = listener.accept().unwrap();
4017 let mut reader = BufReader::new(socket.try_clone().unwrap());
4018 loop {
4019 let mut line = String::new();
4020 reader.read_line(&mut line).unwrap();
4021 if line == "\r\n" {
4022 break;
4023 }
4024 assert!(!line.is_empty());
4025 }
4026 socket.write_all(if body_failure {
4027 b"HTTP/1.1 200 OK\r\nContent-Length: 999\r\nConnection: close\r\n\r\nshort"
4028 } else {
4029 b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
4030 }).unwrap();
4031 });
4032 let mut transport =
4033 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
4034 let mut progress = |_| {};
4035 let error = if signed {
4036 transport.fetch_url(
4037 &format!("{base}/signed?signature=secret"),
4038 1 << 20,
4039 &mut progress,
4040 )
4041 } else {
4042 transport.download_segment(
4043 &SegmentRequest {
4044 segment_id: "sha256:test".into(),
4045 table: "tasks".into(),
4046 requested_scopes_json: "{}".into(),
4047 byte_length: 1 << 20,
4048 },
4049 &mut progress,
4050 )
4051 }
4052 .unwrap_err();
4053 assert_eq!(error.code, "sync.transport_failed");
4054 assert_eq!(error.message, "segment transfer failed");
4055 let details = error.details.unwrap();
4056 assert!(details.get("path").is_none());
4057 assert!(details.get("causeMessage").is_none());
4058 assert_eq!(
4059 details["causeKind"],
4060 if body_failure { "body" } else { "status" }
4061 );
4062 assert_eq!(details["httpStatus"], if body_failure { 200 } else { 403 });
4063 server.join().unwrap();
4064 }
4065 }
4066 }
4067
4068 #[test]
4069 fn blob_status_classification_uses_the_actual_http_response() {
4070 for (status, expected) in [
4071 (404, "blob.not_found"),
4072 (401, "sync.auth_required"),
4073 (403, "blob.forbidden"),
4074 (500, "transport.failed"),
4075 ] {
4076 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4077 let base = format!("http://{}", listener.local_addr().unwrap());
4078 let server = std::thread::spawn(move || {
4079 let (mut socket, _) = listener.accept().unwrap();
4080 let mut reader = BufReader::new(socket.try_clone().unwrap());
4081 loop {
4082 let mut line = String::new();
4083 reader.read_line(&mut line).unwrap();
4084 if line == "\r\n" {
4085 break;
4086 }
4087 assert!(!line.is_empty());
4088 }
4089 let body = r#"{"code":"blob.not_found","message":"https://host/secret-path?secret-query#secret-fragment","retryable":false}"#;
4090 write!(socket, "HTTP/1.1 {status} Error\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();
4091 });
4092 let mut transport =
4093 HostTransport::new_from_config(&serde_json::json!({"baseUrl":base})).unwrap();
4094 let error = transport.blob_download("sha256:test", None).unwrap_err();
4095 assert_eq!(error.code, expected);
4096 assert_eq!(
4097 error.details,
4098 Some(serde_json::json!({
4099 "causeKind": "status",
4100 "httpStatus": status,
4101 "retryable": false
4102 }))
4103 );
4104 assert!(!error.message.contains("secret"));
4105 assert!(!error.message.contains("http"));
4106 server.join().unwrap();
4107 }
4108 }
4109
4110 #[test]
4111 fn refused_segment_connection_has_a_named_code_without_an_invented_status() {
4112 for signed in [false, true] {
4113 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4114 let base = format!("http://{}", listener.local_addr().unwrap());
4115 drop(listener);
4116 let mut transport =
4117 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
4118 let mut progress = |_| {};
4119 let error = if signed {
4120 transport.fetch_url(
4121 &format!("{base}/signed?signature=secret"),
4122 1 << 20,
4123 &mut progress,
4124 )
4125 } else {
4126 transport.download_segment(
4127 &SegmentRequest {
4128 segment_id: "sha256:test".into(),
4129 table: "tasks".into(),
4130 requested_scopes_json: "{}".into(),
4131 byte_length: 1 << 20,
4132 },
4133 &mut progress,
4134 )
4135 }
4136 .unwrap_err();
4137 assert_eq!(error.code, "sync.transport_failed");
4138 assert!(error.details.as_ref().unwrap().get("httpStatus").is_none());
4139 }
4140 }
4141
4142 #[test]
4143 fn segment_http_reports_intermediate_bytes_for_direct_and_signed_urls() {
4144 for signed in [false, true] {
4145 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4146 let base = format!("http://{}", listener.local_addr().unwrap());
4147 let server = std::thread::spawn(move || {
4148 let (mut socket, _) = listener.accept().unwrap();
4149 let mut reader = BufReader::new(socket.try_clone().unwrap());
4150 let mut headers = String::new();
4151 loop {
4152 let mut line = String::new();
4153 reader.read_line(&mut line).unwrap();
4154 if line == "\r\n" {
4155 break;
4156 }
4157 assert!(!line.is_empty());
4158 headers.push_str(&line.to_ascii_lowercase());
4159 }
4160 assert_eq!(headers.contains("authorization: bearer test"), !signed);
4161 assert_eq!(headers.contains("x-syncular-scopes:"), !signed);
4162 let bytes = vec![7; 128 * 1024 + 17];
4163 write!(
4164 socket,
4165 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
4166 bytes.len()
4167 )
4168 .unwrap();
4169 socket.write_all(&bytes).unwrap();
4170 });
4171 let mut transport = HostTransport::from_config(
4172 &serde_json::json!({"baseUrl": base, "headers": {"authorization": "Bearer test"}}),
4173 )
4174 .unwrap();
4175 let mut updates = Vec::new();
4176 let mut progress = |bytes| updates.push(bytes);
4177 let bytes = if signed {
4178 transport
4179 .fetch_url(&format!("{base}/signed"), 1 << 20, &mut progress)
4180 .unwrap()
4181 } else {
4182 transport
4183 .download_segment(
4184 &SegmentRequest {
4185 segment_id: "sha256:test".into(),
4186 table: "tasks".into(),
4187 requested_scopes_json: "{}".into(),
4188 byte_length: 1 << 20,
4189 },
4190 &mut progress,
4191 )
4192 .unwrap()
4193 };
4194 assert_eq!(bytes.len(), 128 * 1024 + 17);
4195 assert!(updates.iter().any(|n| *n > 0 && *n < bytes.len() as u64));
4196 assert_eq!(updates.last(), Some(&(bytes.len() as u64)));
4197 assert!(updates.windows(2).all(|pair| pair[0] < pair[1]));
4198 server.join().unwrap();
4199 }
4200 }
4201
4202 #[test]
4203 fn request_over_cap_is_refused_before_network() {
4204 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4205 listener.set_nonblocking(true).unwrap();
4206 let base = format!("http://{}", listener.local_addr().unwrap());
4207 let mut transport =
4208 HostTransport::from_config(&serde_json::json!({"baseUrl": base, "maxRequestBytes": 4}))
4209 .unwrap();
4210 let error = transport
4211 .blob_upload("sha256:test", &[0u8; 8], None)
4212 .unwrap_err();
4213 assert_eq!(error.code, "transport.request_too_large");
4214 assert_eq!(
4215 error.details,
4216 Some(serde_json::json!({ "limit": 4, "size": 8 }))
4217 );
4218 assert!(
4219 matches!(listener.accept(), Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock),
4220 "the request cap is checked before any connection"
4221 );
4222 }
4223
4224 #[test]
4225 fn credential_bearing_redirects_are_refused_without_contacting_the_destination() {
4226 for credentials in ["headers", "userinfo", "password"] {
4227 for redirects in ["deny", "follow"] {
4228 let destination = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4229 destination.set_nonblocking(true).unwrap();
4230 let location = format!("http://{}/landed", destination.local_addr().unwrap());
4231 let origin = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4232 let host = origin.local_addr().unwrap();
4233 let (base, headers) = match credentials {
4234 "headers" => (
4235 format!("http://{host}"),
4236 serde_json::json!({"authorization": "Bearer secret"}),
4237 ),
4238 "userinfo" => (format!("http://user:secret@{host}"), serde_json::json!({})),
4239 _ => (format!("http://:secret@{host}"), serde_json::json!({})),
4240 };
4241 let server = std::thread::spawn(move || {
4242 let (mut socket, _) = origin.accept().unwrap();
4243 let mut reader = BufReader::new(socket.try_clone().unwrap());
4244 loop {
4245 let mut line = String::new();
4246 reader.read_line(&mut line).unwrap();
4247 if line == "\r\n" {
4248 break;
4249 }
4250 }
4251 let _ = write!(
4252 socket,
4253 "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
4254 );
4255 });
4256 let mut transport = HostTransport::from_config(&serde_json::json!({
4257 "baseUrl": base,
4258 "redirects": redirects,
4259 "headers": headers,
4260 }))
4261 .unwrap();
4262 let error = transport.blob_download("sha256:test", None).unwrap_err();
4263 assert_eq!(
4264 error.code, "transport.redirect",
4265 "{credentials} {redirects}"
4266 );
4267 let details = error.details.unwrap();
4268 assert_eq!(details["causeKind"], "redirect");
4269 assert_eq!(details["httpStatus"], 302);
4270 assert!(
4271 matches!(destination.accept(), Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock),
4272 "{credentials} {redirects}: the destination is never contacted"
4273 );
4274 server.join().unwrap();
4275 }
4276 }
4277 }
4278
4279 #[test]
4280 fn websocket_handshake_redirects_obey_the_redirect_policy() {
4281 for credential in ["headers", "userinfo", "query", "none"] {
4288 let redirects = "deny";
4289 let destination = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4290 let destination_addr = destination.local_addr().unwrap();
4291 let location = format!("ws://{destination_addr}/realtime");
4292 let landed = Arc::new(Mutex::new(Vec::new()));
4296 let landed_writer = Arc::clone(&landed);
4297 let destination_thread = std::thread::spawn(move || loop {
4298 let (mut socket, _) = destination.accept().unwrap();
4299 let mut reader = BufReader::new(socket.try_clone().unwrap());
4300 let mut head = String::new();
4301 loop {
4302 let mut line = String::new();
4303 reader.read_line(&mut line).unwrap();
4304 if line == "\r\n" {
4305 break;
4306 }
4307 head.push_str(&line);
4308 }
4309 let sentinel = head.starts_with("GET /sentinel");
4310 landed_writer.lock().unwrap().push(head);
4311 let _ = write!(
4312 socket,
4313 "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
4314 );
4315 if sentinel {
4316 break;
4317 }
4318 });
4319 let origin = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4320 let origin_addr = origin.local_addr().unwrap();
4321 let (headers, ws_url) = match credential {
4322 "headers" => (serde_json::json!({"authorization": "Bearer secret"}), None),
4323 "userinfo" => (
4324 serde_json::json!({}),
4325 Some(format!("ws://user:secret@{origin_addr}/realtime")),
4326 ),
4327 "query" => (
4328 serde_json::json!({}),
4329 Some(format!("ws://{origin_addr}/realtime?token=secret")),
4330 ),
4331 _ => (serde_json::json!({}), None),
4332 };
4333 let (head_tx, head_rx) = std::sync::mpsc::channel();
4334 let origin_thread = std::thread::spawn(move || {
4335 let (mut socket, _) = origin.accept().unwrap();
4336 let mut reader = BufReader::new(socket.try_clone().unwrap());
4337 let mut head = String::new();
4338 loop {
4339 let mut line = String::new();
4340 reader.read_line(&mut line).unwrap();
4341 if line == "\r\n" {
4342 break;
4343 }
4344 head.push_str(&line);
4345 }
4346 let _ = head_tx.send(head);
4347 let _ = write!(
4348 socket,
4349 "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
4350 );
4351 });
4352 let mut config = serde_json::json!({
4353 "baseUrl": format!("http://{origin_addr}"),
4354 "headers": headers,
4355 "redirects": redirects,
4356 });
4357 if let Some(ws_url) = &ws_url {
4358 config["wsUrl"] = serde_json::json!(ws_url);
4359 }
4360 let mut transport = HostTransport::from_config(&config).unwrap();
4361 let result = transport.realtime_connect();
4362 let mut sentinel = std::net::TcpStream::connect(destination_addr).unwrap();
4365 let _ = write!(sentinel, "GET /sentinel HTTP/1.1\r\nHost: sentinel\r\n\r\n");
4366 destination_thread.join().unwrap();
4367 let landed = landed.lock().unwrap().clone();
4368 origin_thread.join().unwrap();
4369 let error = result.unwrap_err();
4370 assert_eq!(
4371 error.code, "transport.redirect",
4372 "{credential}: a refused handshake redirect surfaces the redirect code"
4373 );
4374 let details = error.details.unwrap();
4375 assert_eq!(details["causeKind"], "redirect");
4376 assert_eq!(details["httpStatus"], 302);
4377 assert_eq!(
4378 landed.len(),
4379 1,
4380 "{credential}: the redirect target is never contacted: {landed:?}"
4381 );
4382 assert!(landed[0].starts_with("GET /sentinel"), "{landed:?}");
4383 let head = head_rx.recv().unwrap();
4384 if credential == "userinfo" {
4385 assert!(
4386 !head.contains("secret"),
4387 "the handshake request line drops URL userinfo"
4388 );
4389 }
4390 if credential == "query" {
4391 assert!(
4392 head.contains("token=secret"),
4393 "the handshake request line carries the realtime query"
4394 );
4395 }
4396 }
4397 }
4398
4399 const GZIP_BOMB: [u8; 548] = [
4402 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xed, 0xc9, 0xb1, 0x11, 0x80,
4403 0x20, 0x10, 0x00, 0xb0, 0x59, 0x1c, 0x80, 0xa1, 0xfe, 0x90, 0xb3, 0x41, 0xf0, 0x7c, 0x2d,
4404 0xd8, 0xde, 0x09, 0xdc, 0x20, 0x69, 0x93, 0x6b, 0xd4, 0xb7, 0xc7, 0x5d, 0xb2, 0x1d, 0x67,
4405 0x1b, 0x4f, 0xb9, 0x62, 0xf5, 0x19, 0xfb, 0x96, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4406 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4407 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4408 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4409 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4410 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4411 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4412 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4413 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4414 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4415 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4416 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4417 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4418 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4419 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4420 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4421 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4422 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4423 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4424 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4425 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4426 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4427 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4428 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4429 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4430 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4431 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4432 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4433 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4434 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4435 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4436 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4437 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0xf8, 0x8b, 0x0f,
4438 0xdb, 0x82, 0xf4, 0xa9, 0x40, 0x0d, 0x03, 0x00,
4439 ];
4440
4441 #[test]
4442 fn decoded_response_cap_counts_decompressed_bytes() {
4443 for cap in [200000u64, 199999] {
4444 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4445 let base = format!("http://{}", listener.local_addr().unwrap());
4446 let server = std::thread::spawn(move || {
4447 let (mut socket, _) = listener.accept().unwrap();
4448 let mut reader = BufReader::new(socket.try_clone().unwrap());
4449 loop {
4450 let mut line = String::new();
4451 reader.read_line(&mut line).unwrap();
4452 if line == "\r\n" {
4453 break;
4454 }
4455 }
4456 let _ = write!(
4457 socket,
4458 "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
4459 GZIP_BOMB.len()
4460 );
4461 let _ = socket.write_all(&GZIP_BOMB);
4462 });
4463 let mut transport = HostTransport::from_config(
4464 &serde_json::json!({"baseUrl": base, "maxResponseBytes": cap}),
4465 )
4466 .unwrap();
4467 let result = transport.fetch_blob_url(&format!("{base}/signed"), None);
4468 if cap == 200000 {
4469 let body = result.expect("a cap equal to the decoded size admits the body");
4470 assert_eq!(body.len(), 200000);
4471 } else {
4472 let error = result.unwrap_err();
4473 assert_eq!(error.code, "transport.response_too_large");
4474 assert_eq!(error.details, Some(serde_json::json!({ "limit": cap })));
4475 }
4476 server.join().unwrap();
4477 }
4478 }
4479
4480 #[test]
4481 fn a_json_http_error_body_reaches_the_caller() {
4482 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4483 let base = format!("http://{}", listener.local_addr().unwrap());
4484 let body = r#"{"code":"sync.invalid_client_id","category":"invalid-request","retryable":false,"recommendedAction":"resetClientId","message":"clientId is bound to a different actor in this partition (§1.5)","details":{"actorId":"actor-b"}}"#;
4485 let server = std::thread::spawn(move || {
4486 let (mut socket, _) = listener.accept().unwrap();
4487 let mut reader = BufReader::new(socket.try_clone().unwrap());
4488 loop {
4489 let mut line = String::new();
4490 reader.read_line(&mut line).unwrap();
4491 if line == "\r\n" {
4492 break;
4493 }
4494 }
4495 let _ = write!(
4496 socket,
4497 "HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
4498 body.len(),
4499 body
4500 );
4501 });
4502 let mut transport =
4503 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
4504 let error = transport.sync(&[]).unwrap_err();
4505 assert_eq!(error.code, "sync.invalid_client_id");
4506 assert_eq!(
4507 error.message,
4508 "clientId is bound to a different actor in this partition (§1.5)"
4509 );
4510 let details = error.details.unwrap();
4511 assert_eq!(details["recommendedAction"], "resetClientId");
4512 assert_eq!(details["category"], "invalid-request");
4513 assert_eq!(details["retryable"], false);
4514 assert_eq!(details["httpStatus"], 400);
4515 assert_eq!(details["actorId"], "actor-b");
4516 server.join().unwrap();
4517 }
4518}
4519
4520#[cfg(test)]
4521mod local_authority_tests {
4522 use super::HostTransport;
4523 use crate::Transport;
4524
4525 #[cfg(not(feature = "local-authority"))]
4526 #[test]
4527 fn a_local_authority_path_without_the_feature_fails_with_the_unconfigured_code() {
4528 let error = HostTransport::from_config(
4529 &serde_json::json!({ "localAuthorityPath": "authority.sqlite" }),
4530 )
4531 .err()
4532 .expect("a build without local-authority refuses localAuthorityPath");
4533 assert!(
4534 error.starts_with("client.local_authority_unconfigured:"),
4535 "{error}"
4536 );
4537 }
4538
4539 #[test]
4540 fn a_config_without_a_local_authority_reports_none() {
4541 let mut transport = HostTransport::from_config(&serde_json::json!({}))
4542 .ok()
4543 .unwrap();
4544 assert_eq!(transport.local_authority_state(), None);
4545 let error = transport.attach_local_authority().unwrap_err();
4546 assert_eq!(error.code, "client.local_authority_unconfigured");
4547 let error = transport.abandon_local_authority_attach().unwrap_err();
4548 assert_eq!(error.code, "client.local_authority_unconfigured");
4549 }
4550
4551 #[cfg(feature = "local-authority")]
4552 mod with_feature {
4553 use serde_json::{json, Map, Value};
4554 use syncular_authority::LocalAuthority;
4555
4556 use super::super::{HostTransport, LocalAuthorityTransport};
4557 use crate::{
4558 ClientLimits, Mutation, SegmentFetch, SegmentRequest, SyncClient, SyncOutcome,
4559 Transport, TransportError, UnitScopes,
4560 };
4561
4562 fn schema() -> Value {
4563 json!({
4564 "version": 1,
4565 "tables": [{
4566 "name": "todos",
4567 "columns": [
4568 { "name": "id", "type": "string", "nullable": false },
4569 { "name": "list_id", "type": "string", "nullable": false }
4570 ],
4571 "primaryKey": "id",
4572 "scopes": [{ "pattern": "list:{list_id}", "column": "list_id" }]
4573 }]
4574 })
4575 }
4576
4577 struct AuthorityServer {
4580 server: LocalAuthority,
4581 syncs: usize,
4582 }
4583
4584 impl Transport for AuthorityServer {
4585 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4586 self.syncs += 1;
4587 self.server.sync(request).map_err(|error| TransportError {
4588 code: error.code,
4589 message: error.message,
4590 details: error.details,
4591 })
4592 }
4593 fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
4594 Err(TransportError::new("sync.transport_failed", "no socket"))
4595 }
4596 fn download_segment(
4597 &mut self,
4598 _request: &SegmentRequest,
4599 _on_progress: &mut dyn FnMut(u64),
4600 ) -> Result<Vec<u8>, TransportError> {
4601 Err(TransportError::new("sync.transport_failed", "no segments"))
4602 }
4603 fn realtime_connect(&mut self) -> Result<(), TransportError> {
4604 Err(TransportError::new("sync.transport_failed", "no socket"))
4605 }
4606 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
4607 Ok(())
4608 }
4609 fn realtime_close(&mut self) -> Result<(), TransportError> {
4610 Ok(())
4611 }
4612 }
4613
4614 #[test]
4615 fn a_configured_path_routes_to_the_authority_and_refuses_what_it_lacks() {
4616 let mut transport =
4617 HostTransport::from_config(&json!({ "localAuthorityPath": ":memory:" }))
4618 .ok()
4619 .unwrap();
4620 assert!(matches!(transport, HostTransport::Local(_)));
4621 assert_eq!(transport.local_authority_state(), None);
4623 assert_eq!(
4624 transport.sync(b"").unwrap_err().code,
4625 "client.local_authority_unconfigured"
4626 );
4627 assert_eq!(
4628 transport
4629 .realtime_connect_for_client("device-1")
4630 .unwrap_err()
4631 .code,
4632 "sync.invalid_request"
4633 );
4634 let refused = transport.download_segments(
4635 &[SegmentFetch::Url {
4636 url: "https://cdn.example.com/s1".to_owned(),
4637 byte_length: 1 << 20,
4638 }],
4639 &mut |_, _| {},
4640 );
4641 assert_eq!(refused.len(), 1);
4642 assert_eq!(
4643 refused[0].as_ref().unwrap_err().code,
4644 "sync.invalid_request"
4645 );
4646 let error = transport.attach_local_authority().unwrap_err();
4648 assert_eq!(error.code, "client.local_authority_unconfigured");
4649 }
4650
4651 #[cfg(feature = "crdt-yjs")]
4654 #[test]
4655 fn a_crdt_commit_through_the_local_authority_is_merged() {
4656 let schema = json!({
4657 "version": 1,
4658 "tables": [{
4659 "name": "notes",
4660 "columns": [
4661 { "name": "id", "type": "string", "nullable": false },
4662 { "name": "list_id", "type": "string", "nullable": false },
4663 { "name": "doc", "type": "crdt", "nullable": true, "crdtType": "yjs-doc" }
4664 ],
4665 "primaryKey": "id",
4666 "scopes": [{ "pattern": "list:{list_id}", "column": "list_id" }]
4667 }]
4668 });
4669 let mut transport =
4670 HostTransport::from_config(&json!({ "localAuthorityPath": ":memory:" }))
4671 .ok()
4672 .unwrap();
4673 transport
4674 .open_local_authority(&schema, None)
4675 .expect("open the local authority");
4676 let mut client = SyncClient::new_with_identity(
4677 Some("device-1".to_owned()),
4678 &schema,
4679 ClientLimits::default(),
4680 )
4681 .expect("client");
4682 client
4683 .subscribe(UnitScopes::from([("list_id".to_owned(), "l1".to_owned())]))
4684 .expect("subscribe");
4685 client
4686 .mutate(vec![Mutation::Upsert {
4687 table: "notes".to_owned(),
4688 values: Map::from_iter([
4689 ("id".to_owned(), json!("n1")),
4690 ("list_id".to_owned(), json!("l1")),
4691 ("doc".to_owned(), Value::Null),
4692 ]),
4693 base_version: None,
4694 }])
4695 .expect("mutate");
4696 client
4697 .crdt_insert_text("notes", "n1", "doc", "text", 0, "éa")
4698 .expect("crdt insert");
4699 let outcome = client.sync_until_idle(&mut transport, None);
4700 assert!(
4701 matches!(outcome, SyncOutcome::Ok(_)),
4702 "{:?}",
4703 outcome.to_json()
4704 );
4705 assert!(client.pending_commit_ids().is_empty());
4706 assert!(client.rejections().is_empty(), "{:?}", client.rejections());
4707 assert_eq!(
4708 client.crdt_text("notes", "n1", "doc", "text").unwrap(),
4709 "éa"
4710 );
4711 }
4712
4713 #[test]
4714 fn the_authority_path_must_differ_from_the_replica_and_be_a_string() {
4715 let mut transport =
4716 HostTransport::from_config(&json!({ "localAuthorityPath": "/data/app.sqlite" }))
4717 .ok()
4718 .unwrap();
4719 let error = transport
4720 .open_local_authority(&schema(), Some("/data/app.sqlite"))
4721 .unwrap_err();
4722 assert_eq!(error.code, "sync.invalid_request");
4723 for path in [json!(""), json!(7)] {
4724 assert!(
4725 HostTransport::from_config(&json!({ "localAuthorityPath": path })).is_err()
4726 );
4727 }
4728 }
4729
4730 #[test]
4731 fn rounds_reach_the_authority_until_attach_then_the_remote_server() {
4732 let schema = schema();
4733 let remote = AuthorityServer {
4734 server: LocalAuthority::open(":memory:", &schema).expect("open the remote"),
4735 syncs: 0,
4736 };
4737 let mut transport = LocalAuthorityTransport::new(":memory:", remote, true);
4738 transport
4739 .open_local_authority(&schema, None)
4740 .expect("open the local authority");
4741 assert_eq!(transport.local_authority_state(), Some("local"));
4742
4743 let mut client = SyncClient::new_with_identity(
4744 Some("device-1".to_owned()),
4745 &schema,
4746 ClientLimits::default(),
4747 )
4748 .expect("client");
4749 client
4750 .subscribe(UnitScopes::from([("list_id".to_owned(), "l1".to_owned())]))
4751 .expect("subscribe");
4752 client
4753 .mutate(vec![Mutation::Upsert {
4754 table: "todos".to_owned(),
4755 values: Map::from_iter([
4756 ("id".to_owned(), json!("t1")),
4757 ("list_id".to_owned(), json!("l1")),
4758 ]),
4759 base_version: None,
4760 }])
4761 .expect("mutate");
4762 let outcome = client.sync_until_idle(&mut transport, None);
4763 assert!(
4764 matches!(outcome, SyncOutcome::Ok(_)),
4765 "{:?}",
4766 outcome.to_json()
4767 );
4768 assert!(client.pending_commit_ids().is_empty());
4769 assert_eq!(
4770 transport.remote.syncs, 0,
4771 "the local authority served the round"
4772 );
4773 let error = client.connect_realtime(&mut transport).unwrap_err();
4774 assert!(error.starts_with("sync.invalid_request:"), "{error}");
4775
4776 transport.attach_local_authority().expect("attach");
4777 assert_eq!(transport.local_authority_state(), Some("attached"));
4778 let uploads = transport.remote.syncs;
4779 assert!(uploads > 0, "attach uploaded through the remote transport");
4780
4781 let outcome = client.sync_until_idle(&mut transport, None);
4784 assert!(
4785 matches!(outcome, SyncOutcome::Ok(_)),
4786 "{:?}",
4787 outcome.to_json()
4788 );
4789 assert!(transport.remote.syncs > uploads);
4790 let rows = client.read_rows("todos").expect("rows");
4791 assert_eq!(rows.len(), 1);
4792 assert_eq!(rows[0].row_id, "t1");
4793 assert!(rows[0].version >= 1);
4794 }
4795 }
4796}