1use std::sync::atomic::{AtomicU64, Ordering};
24use std::sync::{mpsc, Arc, Mutex};
25use std::time::{Duration, SystemTime};
26
27use crate::{
28 BlobDownload, BlobUploadGrant, SegmentFetch, SegmentRequest, Transport, TransportError,
29};
30
31#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
33pub enum RedirectPolicy {
34 #[default]
39 Deny,
40 Follow,
50}
51
52#[derive(Debug, Clone, Default)]
55pub struct HostTransportPolicy {
56 pub request_timeout: Option<Duration>,
58 pub round_deadline: Option<Duration>,
63 pub max_request_bytes: Option<u64>,
66 pub max_response_bytes: Option<u64>,
69 pub redirects: RedirectPolicy,
70 pub realtime_heartbeat: Option<Duration>,
74}
75
76impl HostTransportPolicy {
77 pub fn validate(&self) -> Result<(), String> {
81 let now = std::time::Instant::now();
82 if let Some(timeout) = self.request_timeout {
83 if timeout.is_zero() || now.checked_add(timeout).is_none() {
84 return Err(
85 "sync.invalid_request: transport config requestTimeoutMs is out of range"
86 .to_owned(),
87 );
88 }
89 }
90 if let Some(deadline) = self.round_deadline {
91 if deadline.is_zero() || now.checked_add(deadline).is_none() {
92 return Err(
93 "sync.invalid_request: transport config roundDeadlineMs is out of range"
94 .to_owned(),
95 );
96 }
97 }
98 if let Some(interval) = self.realtime_heartbeat {
99 if interval.is_zero()
100 || interval
101 .checked_mul(2)
102 .and_then(|silence| now.checked_add(silence))
103 .is_none()
104 {
105 return Err(
106 "sync.invalid_request: transport config realtimeHeartbeatMs is out of range"
107 .to_owned(),
108 );
109 }
110 }
111 if self.max_request_bytes == Some(0) {
112 return Err(
113 "sync.invalid_request: transport config maxRequestBytes must be a positive integer"
114 .to_owned(),
115 );
116 }
117 if self.max_response_bytes == Some(0) {
118 return Err("sync.invalid_request: transport config maxResponseBytes must be a positive integer".to_owned());
119 }
120 Ok(())
121 }
122
123 pub fn from_config(config: &serde_json::Value) -> Result<Self, String> {
126 let bound = |key: &str, message: &'static str| -> Result<Option<u64>, String> {
127 match config.get(key) {
128 None | Some(serde_json::Value::Null) => Ok(None),
129 Some(value) => match value.as_u64() {
130 Some(number) if number >= 1 => Ok(Some(number)),
131 _ => Err(message.to_owned()),
132 },
133 }
134 };
135 let redirects = match config.get("redirects") {
136 None | Some(serde_json::Value::Null) => RedirectPolicy::Deny,
137 Some(serde_json::Value::String(value)) if value == "deny" => RedirectPolicy::Deny,
138 Some(serde_json::Value::String(value)) if value == "follow" => RedirectPolicy::Follow,
139 Some(_) => return Err(
140 "sync.invalid_request: transport config redirects must be \"deny\" or \"follow\""
141 .to_owned(),
142 ),
143 };
144 let policy = Self {
145 request_timeout: bound(
146 "requestTimeoutMs",
147 "sync.invalid_request: transport config requestTimeoutMs must be a positive integer",
148 )?
149 .map(Duration::from_millis),
150 round_deadline: bound(
151 "roundDeadlineMs",
152 "sync.invalid_request: transport config roundDeadlineMs must be a positive integer",
153 )?
154 .map(Duration::from_millis),
155 max_request_bytes: bound(
156 "maxRequestBytes",
157 "sync.invalid_request: transport config maxRequestBytes must be a positive integer",
158 )?,
159 max_response_bytes: bound(
160 "maxResponseBytes",
161 "sync.invalid_request: transport config maxResponseBytes must be a positive integer",
162 )?,
163 redirects,
164 realtime_heartbeat: bound(
165 "realtimeHeartbeatMs",
166 "sync.invalid_request: transport config realtimeHeartbeatMs must be a positive integer",
167 )?
168 .map(Duration::from_millis),
169 };
170 policy.validate()?;
171 Ok(policy)
172 }
173}
174
175pub enum Inbound {
177 Text(String),
178 Binary(Vec<u8>),
179 Disconnected(TransportError),
182}
183
184#[derive(Debug, Clone, Copy, PartialEq, Eq)]
187pub struct RealtimeReceipt {
188 pub received_at: SystemTime,
189 pub bytes: usize,
190 pub kind: RealtimeReceiptKind,
191}
192
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
194pub enum RealtimeReceiptKind {
195 Text,
196 Delta,
197 Round,
198 OtherBinary,
199 Ping,
200 Pong,
201 Close,
202}
203
204pub const INBOUND_BUFFER_LIMIT_BYTES: usize = 16 * 1024 * 1024;
208
209#[derive(Default)]
211struct InboundFrames {
212 list: Vec<Inbound>,
213 bytes: usize,
214}
215
216pub struct InboundBuffer {
220 frames: Mutex<InboundFrames>,
221 notify: Option<Arc<dyn Fn() + Send + Sync>>,
222 receipts: Mutex<Option<mpsc::SyncSender<RealtimeReceipt>>>,
223 dropped_receipts: AtomicU64,
224}
225
226impl Default for InboundBuffer {
227 fn default() -> Self {
228 Self {
229 frames: Mutex::new(InboundFrames::default()),
230 notify: None,
231 receipts: Mutex::new(None),
232 dropped_receipts: AtomicU64::new(0),
233 }
234 }
235}
236
237impl InboundBuffer {
238 pub fn with_notify(notify: Arc<dyn Fn() + Send + Sync>) -> Self {
239 Self {
240 frames: Mutex::new(InboundFrames::default()),
241 notify: Some(notify),
242 receipts: Mutex::new(None),
243 dropped_receipts: AtomicU64::new(0),
244 }
245 }
246
247 pub fn push(&self, frame: Inbound) {
248 {
249 let mut frames = self.frames.lock().expect("inbound lock");
250 frames.bytes += match &frame {
251 Inbound::Text(text) => text.len(),
252 Inbound::Binary(bytes) => bytes.len(),
253 Inbound::Disconnected(_) => 0,
254 };
255 frames.list.push(frame);
256 }
257 if let Some(notify) = &self.notify {
258 notify();
259 }
260 }
261
262 fn take(&self) -> Vec<Inbound> {
263 std::mem::take(&mut *self.frames.lock().expect("inbound lock")).list
264 }
265
266 #[cfg(feature = "native-transport")]
268 fn overflowed(&self) -> bool {
269 self.frames.lock().expect("inbound lock").bytes > INBOUND_BUFFER_LIMIT_BYTES
270 }
271
272 #[cfg(feature = "native-transport")]
273 fn observe(&self, kind: RealtimeReceiptKind, bytes: usize) {
274 if let Some(sender) = self.receipts.lock().expect("receipt lock").as_ref() {
275 let receipt = RealtimeReceipt {
276 received_at: SystemTime::now(),
277 bytes,
278 kind,
279 };
280 if sender.try_send(receipt).is_err() {
281 self.dropped_receipts.fetch_add(1, Ordering::Relaxed);
282 }
283 }
284 }
285}
286
287#[allow(clippy::large_enum_variant)]
290pub enum HostTransport {
291 Null {
293 signed_urls: bool,
294 inbound: Arc<InboundBuffer>,
295 },
296 #[cfg(feature = "native-transport")]
297 Native(native::NativeTransport),
298 #[cfg(feature = "local-authority")]
300 Local(Box<LocalAuthorityTransport<HostTransport>>),
301}
302
303impl HostTransport {
304 pub fn from_config(config: &serde_json::Value) -> Result<Self, String> {
307 Self::from_config_with_notify(config, None)
308 }
309
310 pub fn from_config_with_notify(
314 config: &serde_json::Value,
315 notify: Option<Arc<dyn Fn() + Send + Sync>>,
316 ) -> Result<Self, String> {
317 let policy = HostTransportPolicy::from_config(config)?;
318 Self::from_config_with_policy(config, policy, notify)
319 }
320
321 pub fn from_config_with_policy(
325 config: &serde_json::Value,
326 policy: HostTransportPolicy,
327 notify: Option<Arc<dyn Fn() + Send + Sync>>,
328 ) -> Result<Self, String> {
329 let local_authority_path = match config.get("localAuthorityPath") {
330 None | Some(serde_json::Value::Null) => None,
331 Some(serde_json::Value::String(path)) if !path.is_empty() => Some(path.clone()),
332 Some(_) => return Err("localAuthorityPath must be a non-empty string".to_owned()),
333 };
334 #[cfg(not(feature = "local-authority"))]
335 if local_authority_path.is_some() {
336 return Err(format!(
337 "{}: this build has no local authority (rebuild with --features local-authority)",
338 crate::LOCAL_AUTHORITY_UNCONFIGURED_CODE
339 ));
340 }
341 let remote = match config.get("baseUrl").and_then(|v| v.as_str()) {
342 #[cfg(feature = "native-transport")]
343 Some(base_url) => HostTransport::Native(native::NativeTransport::with_policy(
344 base_url, config, policy, notify,
345 )?),
346 _ => {
347 #[cfg(not(feature = "native-transport"))]
348 {
349 policy.validate()?;
350 if config.get("baseUrl").is_some() {
351 return Err(
352 "this build has no native transport (rebuild with --features native-transport)"
353 .to_owned(),
354 );
355 }
356 }
357 HostTransport::Null {
358 signed_urls: false,
359 inbound: Arc::new(match notify {
360 Some(notify) => InboundBuffer::with_notify(notify),
361 None => InboundBuffer::default(),
362 }),
363 }
364 }
365 };
366 #[cfg(feature = "local-authority")]
367 if let Some(path) = local_authority_path {
368 let remote_configured = !matches!(remote, HostTransport::Null { .. });
369 return Ok(HostTransport::Local(Box::new(
370 LocalAuthorityTransport::new(path, remote, remote_configured),
371 )));
372 }
373 Ok(remote)
374 }
375
376 pub fn fork_round(&self) -> Self {
379 match self {
380 Self::Null {
381 signed_urls,
382 inbound,
383 } => Self::Null {
384 signed_urls: *signed_urls,
385 inbound: inbound.clone(),
386 },
387 #[cfg(feature = "native-transport")]
388 Self::Native(t) => Self::Native(t.fork_round()),
389 #[cfg(feature = "local-authority")]
390 Self::Local(t) => Self::Local(Box::new(t.fork_round())),
391 }
392 }
393
394 pub fn set_signed_urls(&mut self, value: bool) {
395 match self {
396 HostTransport::Null { signed_urls, .. } => *signed_urls = value,
397 #[cfg(feature = "native-transport")]
398 HostTransport::Native(t) => t.signed_urls = value,
399 #[cfg(feature = "local-authority")]
400 HostTransport::Local(t) => t.remote.set_signed_urls(value),
401 }
402 }
403
404 pub fn set_headers(&mut self, headers: Vec<(String, String)>) {
407 match self {
408 HostTransport::Null { .. } => drop(headers),
409 #[cfg(feature = "native-transport")]
410 HostTransport::Native(t) => t.set_headers(headers),
411 #[cfg(feature = "local-authority")]
412 HostTransport::Local(t) => t.remote.set_headers(headers),
413 }
414 }
415
416 pub fn set_policy(&mut self, policy: HostTransportPolicy) -> Result<(), String> {
421 policy.validate()?;
422 match self {
423 HostTransport::Null { .. } => {}
424 #[cfg(feature = "native-transport")]
425 HostTransport::Native(t) => t.set_policy(policy)?,
426 #[cfg(feature = "local-authority")]
427 HostTransport::Local(t) => t.remote.set_policy(policy)?,
428 }
429 Ok(())
430 }
431
432 pub fn take_inbound(&mut self) -> Vec<Inbound> {
434 match self {
435 HostTransport::Null { inbound, .. } => inbound.take(),
436 #[cfg(feature = "native-transport")]
437 HostTransport::Native(t) => t.inbound.take(),
438 #[cfg(feature = "local-authority")]
439 HostTransport::Local(t) => t.remote.take_inbound(),
440 }
441 }
442
443 pub fn set_realtime_receipt_observer(
448 &mut self,
449 sender: Option<mpsc::SyncSender<RealtimeReceipt>>,
450 ) {
451 let inbound = match self {
452 HostTransport::Null { inbound, .. } => inbound,
453 #[cfg(feature = "native-transport")]
454 HostTransport::Native(t) => &t.inbound,
455 #[cfg(feature = "local-authority")]
456 HostTransport::Local(t) => {
457 t.remote.set_realtime_receipt_observer(sender);
458 return;
459 }
460 };
461 let mut current = inbound.receipts.lock().expect("receipt lock");
462 *current = sender;
463 inbound.dropped_receipts.store(0, Ordering::Relaxed);
464 }
465
466 pub fn take_dropped_realtime_receipts(&mut self) -> u64 {
468 let inbound = match self {
469 HostTransport::Null { inbound, .. } => inbound,
470 #[cfg(feature = "native-transport")]
471 HostTransport::Native(t) => &t.inbound,
472 #[cfg(feature = "local-authority")]
473 HostTransport::Local(t) => return t.remote.take_dropped_realtime_receipts(),
474 };
475 inbound.dropped_receipts.swap(0, Ordering::Relaxed)
476 }
477
478 pub fn shutdown(&mut self) {
481 match self {
482 HostTransport::Null { .. } => {}
483 #[cfg(feature = "native-transport")]
484 HostTransport::Native(t) => t.shutdown(),
485 #[cfg(feature = "local-authority")]
486 HostTransport::Local(t) => t.shutdown(),
487 }
488 }
489}
490
491fn unavailable(op: &str) -> TransportError {
492 TransportError::new(
493 "transport.unavailable",
494 format!("{op} needs the native transport (build with --features native-transport)"),
495 )
496}
497
498#[cfg_attr(not(feature = "native-transport"), allow(unused_variables))]
501impl Transport for HostTransport {
502 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
503 match self {
504 HostTransport::Null { .. } => Err(unavailable("sync")),
505 #[cfg(feature = "native-transport")]
506 HostTransport::Native(t) => t.sync(request),
507 #[cfg(feature = "local-authority")]
508 HostTransport::Local(t) => t.sync(request),
509 }
510 }
511
512 fn remote_operation(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
513 match self {
514 HostTransport::Null { .. } => Err(unavailable("remoteOperation")),
515 #[cfg(feature = "native-transport")]
516 HostTransport::Native(t) => t.remote_operation(request),
517 #[cfg(feature = "local-authority")]
518 HostTransport::Local(t) => t.remote_operation(request),
519 }
520 }
521
522 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
523 match self {
524 HostTransport::Null { .. } => Err(unavailable("realtimeSync")),
525 #[cfg(feature = "native-transport")]
526 HostTransport::Native(t) => t.realtime_sync(request),
527 #[cfg(feature = "local-authority")]
528 HostTransport::Local(t) => t.realtime_sync(request),
529 }
530 }
531
532 fn download_segment(
533 &mut self,
534 request: &SegmentRequest,
535 on_progress: &mut dyn FnMut(u64),
536 ) -> Result<Vec<u8>, TransportError> {
537 match self {
538 HostTransport::Null { .. } => Err(unavailable("downloadSegment")),
539 #[cfg(feature = "native-transport")]
540 HostTransport::Native(t) => t.download_segment(request, on_progress),
541 #[cfg(feature = "local-authority")]
542 HostTransport::Local(t) => t.download_segment(request, on_progress),
543 }
544 }
545
546 fn supports_url_fetch(&self) -> bool {
547 match self {
548 HostTransport::Null { signed_urls, .. } => *signed_urls,
549 #[cfg(feature = "native-transport")]
550 HostTransport::Native(t) => t.signed_urls,
551 #[cfg(feature = "local-authority")]
552 HostTransport::Local(t) => t.supports_url_fetch(),
553 }
554 }
555
556 fn download_segments(
557 &mut self,
558 fetches: &[SegmentFetch],
559 on_progress: &mut (dyn FnMut(usize, u64) + Send),
560 ) -> Vec<Result<Vec<u8>, TransportError>> {
561 match self {
562 HostTransport::Null { .. } => fetches
563 .first()
564 .map(|_| Err(unavailable("downloadSegments")))
565 .into_iter()
566 .collect(),
567 #[cfg(feature = "native-transport")]
568 HostTransport::Native(t) => t.download_segments(fetches, on_progress),
569 #[cfg(feature = "local-authority")]
570 HostTransport::Local(t) => t.download_segments(fetches, on_progress),
571 }
572 }
573
574 fn fetch_url(
575 &mut self,
576 url: &str,
577 byte_length: u64,
578 on_progress: &mut dyn FnMut(u64),
579 ) -> Result<Vec<u8>, TransportError> {
580 match self {
581 HostTransport::Null { .. } => Err(unavailable("fetchUrl")),
582 #[cfg(feature = "native-transport")]
583 HostTransport::Native(t) => t.fetch_url(url, byte_length, on_progress),
584 #[cfg(feature = "local-authority")]
585 HostTransport::Local(t) => t.fetch_url(url, byte_length, on_progress),
586 }
587 }
588
589 fn blob_upload(
590 &mut self,
591 blob_id: &str,
592 bytes: &[u8],
593 media_type: Option<&str>,
594 ) -> Result<(), TransportError> {
595 match self {
596 HostTransport::Null { .. } => Err(unavailable("blobUpload")),
597 #[cfg(feature = "native-transport")]
598 HostTransport::Native(t) => t.blob_upload(blob_id, bytes, media_type),
599 #[cfg(feature = "local-authority")]
600 HostTransport::Local(t) => t.blob_upload(blob_id, bytes, media_type),
601 }
602 }
603
604 fn blob_download(
605 &mut self,
606 blob_id: &str,
607 byte_length: Option<u64>,
608 ) -> Result<BlobDownload, TransportError> {
609 match self {
610 HostTransport::Null { .. } => Err(unavailable("blobDownload")),
611 #[cfg(feature = "native-transport")]
612 HostTransport::Native(t) => t.blob_download(blob_id, byte_length),
613 #[cfg(feature = "local-authority")]
614 HostTransport::Local(t) => t.blob_download(blob_id, byte_length),
615 }
616 }
617
618 fn fetch_blob_url(
619 &mut self,
620 url: &str,
621 byte_length: Option<u64>,
622 ) -> Result<Vec<u8>, TransportError> {
623 match self {
624 HostTransport::Null { .. } => Err(unavailable("fetchBlobUrl")),
625 #[cfg(feature = "native-transport")]
626 HostTransport::Native(t) => t.fetch_blob_url(url, byte_length),
627 #[cfg(feature = "local-authority")]
628 HostTransport::Local(t) => t.fetch_blob_url(url, byte_length),
629 }
630 }
631
632 fn blob_upload_grant(
633 &mut self,
634 blob_id: &str,
635 byte_length: u64,
636 media_type: Option<&str>,
637 ) -> Result<BlobUploadGrant, TransportError> {
638 match self {
639 HostTransport::Null { .. } => Ok(BlobUploadGrant::None),
642 #[cfg(feature = "native-transport")]
643 HostTransport::Native(t) => t.blob_upload_grant(blob_id, byte_length, media_type),
644 #[cfg(feature = "local-authority")]
645 HostTransport::Local(t) => t.blob_upload_grant(blob_id, byte_length, media_type),
646 }
647 }
648
649 fn blob_put_url(
650 &mut self,
651 blob_id: &str,
652 url: &str,
653 bytes: &[u8],
654 media_type: Option<&str>,
655 ) -> Result<(), TransportError> {
656 match self {
657 HostTransport::Null { .. } => Err(unavailable("blobPutUrl")),
658 #[cfg(feature = "native-transport")]
659 HostTransport::Native(t) => t.blob_put_url(blob_id, url, bytes, media_type),
660 #[cfg(feature = "local-authority")]
661 HostTransport::Local(t) => t.blob_put_url(blob_id, url, bytes, media_type),
662 }
663 }
664
665 fn realtime_connect(&mut self, client_id: &str) -> Result<(), TransportError> {
666 match self {
667 HostTransport::Null { .. } => Err(unavailable("realtimeConnect")),
668 #[cfg(feature = "native-transport")]
669 HostTransport::Native(t) => t.realtime_connect(client_id),
670 #[cfg(feature = "local-authority")]
671 HostTransport::Local(t) => t.realtime_connect(client_id),
672 }
673 }
674
675 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
676 match self {
677 HostTransport::Null { .. } => Err(unavailable("realtimeSend")),
678 #[cfg(feature = "native-transport")]
679 HostTransport::Native(t) => t.realtime_send(text),
680 #[cfg(feature = "local-authority")]
681 HostTransport::Local(t) => t.realtime_send(text),
682 }
683 }
684
685 fn realtime_close(&mut self) -> Result<(), TransportError> {
686 match self {
687 HostTransport::Null { .. } => Ok(()),
688 #[cfg(feature = "native-transport")]
689 HostTransport::Native(t) => t.realtime_close(),
690 #[cfg(feature = "local-authority")]
691 HostTransport::Local(t) => t.realtime_close(),
692 }
693 }
694
695 fn round_deadline(&self) -> Option<std::time::Duration> {
696 match self {
697 HostTransport::Null { .. } => None,
698 #[cfg(feature = "native-transport")]
699 HostTransport::Native(t) => t.round_deadline(),
700 #[cfg(feature = "local-authority")]
701 HostTransport::Local(t) => t.round_deadline(),
702 }
703 }
704
705 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
706 match self {
707 HostTransport::Null { .. } => {}
708 #[cfg(feature = "native-transport")]
709 HostTransport::Native(t) => t.set_round_deadline(deadline),
710 #[cfg(feature = "local-authority")]
711 HostTransport::Local(t) => t.set_round_deadline(deadline),
712 }
713 }
714
715 #[cfg(feature = "local-authority")]
716 fn open_local_authority(
717 &mut self,
718 schema: &serde_json::Value,
719 replica_path: Option<&str>,
720 ) -> Result<(), TransportError> {
721 match self {
722 HostTransport::Local(t) => t.open_local_authority(schema, replica_path),
723 _ => Ok(()),
724 }
725 }
726
727 #[cfg(feature = "local-authority")]
728 fn local_authority_state(&self) -> Option<&'static str> {
729 match self {
730 HostTransport::Local(t) => t.local_authority_state(),
731 _ => None,
732 }
733 }
734
735 #[cfg(feature = "local-authority")]
736 fn attach_local_authority(&mut self) -> Result<(), TransportError> {
737 match self {
738 HostTransport::Local(t) => t.attach_local_authority(),
739 _ => Err(local::unconfigured("this transport has no local authority")),
740 }
741 }
742
743 #[cfg(feature = "local-authority")]
744 fn abandon_local_authority_attach(&mut self) -> Result<(), TransportError> {
745 match self {
746 HostTransport::Local(t) => t.abandon_local_authority_attach(),
747 _ => Err(local::unconfigured("this transport has no local authority")),
748 }
749 }
750}
751
752#[cfg(feature = "local-authority")]
753pub use local::LocalAuthorityTransport;
754
755#[cfg(feature = "local-authority")]
756mod local {
757 use std::sync::{Arc, Mutex};
765
766 use syncular_authority::{AuthorityError, AuthorityState, LocalAuthority};
767
768 use super::HostTransport;
769 use crate::{
770 BlobDownload, BlobUploadGrant, SegmentFetch, SegmentRequest, Transport, TransportError,
771 LOCAL_AUTHORITY_UNCONFIGURED_CODE,
772 };
773
774 pub(super) fn unconfigured(message: &'static str) -> TransportError {
775 TransportError::new(LOCAL_AUTHORITY_UNCONFIGURED_CODE, message)
776 }
777
778 fn transport_error(error: AuthorityError) -> TransportError {
779 TransportError {
780 code: error.code,
781 message: error.message,
782 details: error.details,
783 }
784 }
785
786 fn authority_error(error: TransportError) -> AuthorityError {
787 AuthorityError {
788 code: error.code,
789 message: error.message,
790 details: error.details,
791 }
792 }
793
794 fn refused(message: &'static str) -> TransportError {
795 TransportError::new("sync.invalid_request", message)
796 }
797
798 pub struct LocalAuthorityTransport<R> {
800 path: String,
801 authority: Arc<Mutex<Option<LocalAuthority>>>,
804 pub(super) remote: R,
805 remote_configured: bool,
808 }
809
810 impl<R> LocalAuthorityTransport<R> {
811 pub fn new(path: impl Into<String>, remote: R, remote_configured: bool) -> Self {
815 Self {
816 path: path.into(),
817 authority: Arc::new(Mutex::new(None)),
818 remote,
819 remote_configured,
820 }
821 }
822
823 fn local_active(&self) -> bool {
824 self.authority
825 .lock()
826 .expect("local authority lock")
827 .as_ref()
828 .is_none_or(|authority| authority.state() != AuthorityState::Attached)
829 }
830 }
831
832 impl LocalAuthorityTransport<HostTransport> {
833 pub(super) fn fork_round(&self) -> Self {
834 Self {
835 path: self.path.clone(),
836 authority: self.authority.clone(),
837 remote: self.remote.fork_round(),
838 remote_configured: self.remote_configured,
839 }
840 }
841
842 pub(super) fn shutdown(&mut self) {
843 self.remote.shutdown();
844 *self.authority.lock().expect("local authority lock") = None;
845 }
846 }
847
848 impl<R: Transport> Transport for LocalAuthorityTransport<R> {
849 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
850 let mut authority = self.authority.lock().expect("local authority lock");
851 match authority.as_mut() {
852 None => Err(unconfigured(
853 "the local authority opens with create; create a client first",
854 )),
855 Some(local) if local.state() != AuthorityState::Attached => {
856 local.sync(request).map_err(transport_error)
857 }
858 Some(_) => {
859 drop(authority);
860 self.remote.sync(request)
861 }
862 }
863 }
864
865 fn remote_operation(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
866 self.remote.remote_operation(request)
867 }
868
869 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
870 if self.local_active() {
871 return Err(refused("the local authority has no realtime channel"));
872 }
873 self.remote.realtime_sync(request)
874 }
875
876 fn download_segment(
877 &mut self,
878 request: &SegmentRequest,
879 on_progress: &mut dyn FnMut(u64),
880 ) -> Result<Vec<u8>, TransportError> {
881 if self.local_active() {
882 return Err(refused("the local authority has no segment endpoint"));
883 }
884 self.remote.download_segment(request, on_progress)
885 }
886
887 fn download_segments(
888 &mut self,
889 fetches: &[SegmentFetch],
890 on_progress: &mut (dyn FnMut(usize, u64) + Send),
891 ) -> Vec<Result<Vec<u8>, TransportError>> {
892 if self.local_active() {
893 return fetches
894 .first()
895 .map(|_| Err(refused("the local authority has no segment endpoint")))
896 .into_iter()
897 .collect();
898 }
899 self.remote.download_segments(fetches, on_progress)
900 }
901
902 fn supports_url_fetch(&self) -> bool {
903 self.remote.supports_url_fetch()
904 }
905
906 fn fetch_url(
907 &mut self,
908 url: &str,
909 byte_length: u64,
910 on_progress: &mut dyn FnMut(u64),
911 ) -> Result<Vec<u8>, TransportError> {
912 if self.local_active() {
913 return Err(refused("the local authority has no segment endpoint"));
914 }
915 self.remote.fetch_url(url, byte_length, on_progress)
916 }
917
918 fn blob_upload(
919 &mut self,
920 blob_id: &str,
921 bytes: &[u8],
922 media_type: Option<&str>,
923 ) -> Result<(), TransportError> {
924 self.remote.blob_upload(blob_id, bytes, media_type)
925 }
926
927 fn blob_download(
928 &mut self,
929 blob_id: &str,
930 byte_length: Option<u64>,
931 ) -> Result<BlobDownload, TransportError> {
932 self.remote.blob_download(blob_id, byte_length)
933 }
934
935 fn fetch_blob_url(
936 &mut self,
937 url: &str,
938 byte_length: Option<u64>,
939 ) -> Result<Vec<u8>, TransportError> {
940 self.remote.fetch_blob_url(url, byte_length)
941 }
942
943 fn blob_upload_grant(
944 &mut self,
945 blob_id: &str,
946 byte_length: u64,
947 media_type: Option<&str>,
948 ) -> Result<BlobUploadGrant, TransportError> {
949 self.remote
950 .blob_upload_grant(blob_id, byte_length, media_type)
951 }
952
953 fn blob_put_url(
954 &mut self,
955 blob_id: &str,
956 url: &str,
957 bytes: &[u8],
958 media_type: Option<&str>,
959 ) -> Result<(), TransportError> {
960 self.remote.blob_put_url(blob_id, url, bytes, media_type)
961 }
962
963 fn realtime_connect(&mut self, client_id: &str) -> Result<(), TransportError> {
964 if self.local_active() {
965 return Err(refused("the local authority has no realtime channel"));
966 }
967 self.remote.realtime_connect(client_id)
968 }
969
970 fn round_deadline(&self) -> Option<std::time::Duration> {
971 self.remote.round_deadline()
972 }
973
974 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
975 self.remote.set_round_deadline(deadline);
976 }
977
978 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
979 self.remote.realtime_send(text)
980 }
981
982 fn realtime_close(&mut self) -> Result<(), TransportError> {
983 self.remote.realtime_close()
984 }
985
986 fn open_local_authority(
987 &mut self,
988 schema: &serde_json::Value,
989 replica_path: Option<&str>,
990 ) -> Result<(), TransportError> {
991 if self.path != ":memory:" && replica_path == Some(self.path.as_str()) {
992 return Err(TransportError::new(
993 "sync.invalid_request",
994 "localAuthorityPath must name a file other than the replica's dbPath",
995 ));
996 }
997 let mut authority = self.authority.lock().expect("local authority lock");
998 *authority = None;
1000 *authority = Some(LocalAuthority::open(&self.path, schema).map_err(transport_error)?);
1001 Ok(())
1002 }
1003
1004 fn local_authority_state(&self) -> Option<&'static str> {
1005 self.authority
1006 .lock()
1007 .expect("local authority lock")
1008 .as_ref()
1009 .map(|authority| match authority.state() {
1010 AuthorityState::Local => "local",
1011 AuthorityState::Attaching => "attaching",
1012 AuthorityState::Attached => "attached",
1013 })
1014 }
1015
1016 fn attach_local_authority(&mut self) -> Result<(), TransportError> {
1017 if !self.remote_configured {
1018 return Err(unconfigured(
1019 "attachLocalAuthority needs a baseUrl for the remote server",
1020 ));
1021 }
1022 let remote = &mut self.remote;
1023 let mut authority = self.authority.lock().expect("local authority lock");
1024 authority
1025 .as_mut()
1026 .ok_or_else(|| {
1027 unconfigured("the local authority opens with create; create a client first")
1028 })?
1029 .attach(&mut |request: &[u8]| remote.sync(request).map_err(authority_error))
1030 .map_err(transport_error)
1031 }
1032
1033 fn abandon_local_authority_attach(&mut self) -> Result<(), TransportError> {
1034 if !self.remote_configured {
1035 return Err(unconfigured(
1036 "abandonLocalAuthorityAttach needs a baseUrl for the remote server",
1037 ));
1038 }
1039 let remote = &mut self.remote;
1040 let mut authority = self.authority.lock().expect("local authority lock");
1041 authority
1042 .as_mut()
1043 .ok_or_else(|| {
1044 unconfigured("the local authority opens with create; create a client first")
1045 })?
1046 .abandon_attach(Some(&mut |request: &[u8]| {
1047 remote.sync(request).map_err(authority_error)
1048 }))
1049 .map_err(transport_error)
1050 }
1051 }
1052}
1053
1054#[cfg(feature = "native-transport")]
1055mod native {
1056 use std::io::{self, Read, Write};
1068 use std::net::{TcpStream, ToSocketAddrs};
1069 use std::sync::atomic::{AtomicBool, Ordering};
1070 use std::sync::{mpsc, Arc, Condvar, Mutex};
1071 use std::thread::JoinHandle;
1072 use std::time::{Duration, Instant};
1073
1074 use polling::{Event, Events, Poller};
1075 use tungstenite::stream::MaybeTlsStream;
1076 use tungstenite::Message;
1077 use ureq::unversioned::resolver::DefaultResolver;
1078 use ureq::unversioned::transport::{
1079 time::Duration as UreqDuration, ConnectionDetails, Connector, DefaultConnector,
1080 NextTimeout, Transport as UreqTransport,
1081 };
1082 use ureq::Timeout as UreqTimeout;
1083
1084 use super::{HostTransportPolicy, Inbound, InboundBuffer, RealtimeReceiptKind, RedirectPolicy};
1085 use crate::{
1086 BlobDownload, BlobUploadGrant, RealtimeRound, RoundInbound, SegmentFetch, SegmentRequest,
1087 Transport, TransportError,
1088 };
1089
1090 struct SocketRegistration {
1092 poller: Arc<Poller>,
1093 stream: TcpStream,
1094 }
1095
1096 impl Drop for SocketRegistration {
1097 fn drop(&mut self) {
1098 let _ = self.poller.delete(&self.stream);
1099 }
1100 }
1101
1102 #[derive(Clone)]
1103 struct SocketSender {
1104 queue: mpsc::SyncSender<Outgoing>,
1105 poller: Arc<Poller>,
1106 }
1107
1108 struct Outgoing {
1109 message: Message,
1110 completed: mpsc::Sender<Result<(), TransportError>>,
1111 }
1112
1113 const ROUND_IDLE_TIMEOUT: Duration = Duration::from_secs(30);
1119 const REQUEST_CHUNK_BYTES: usize = 64 * 1024;
1122
1123 fn round_expiry(
1126 last_progress: Instant,
1127 idle: Duration,
1128 ceiling: Option<Instant>,
1129 ) -> (Instant, bool) {
1130 let silent_at = last_progress + idle;
1131 match ceiling {
1132 Some(ceiling) if ceiling < silent_at => (ceiling, true),
1133 _ => (silent_at, false),
1134 }
1135 }
1136 #[derive(Default)]
1143 pub(super) struct RoundChannel {
1144 state: Mutex<RoundState>,
1145 ready: Condvar,
1146 }
1147
1148 #[derive(Default)]
1149 struct RoundState {
1150 round: RealtimeRound,
1151 last_progress: Option<Instant>,
1154 lost: bool,
1156 outcome: Option<Result<Vec<u8>, TransportError>>,
1158 }
1159
1160 impl RoundChannel {
1161 fn begin(&self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1165 let mut state = self.state.lock().expect("round lock");
1166 state.outcome = None;
1167 let framed = state.round.begin(request)?;
1168 state.last_progress = Some(Instant::now());
1169 Ok(framed)
1170 }
1171
1172 fn progressed(&self) {
1174 self.state.lock().expect("round lock").last_progress = Some(Instant::now());
1175 }
1176
1177 fn lost(&self) -> bool {
1180 self.state.lock().expect("round lock").lost
1181 }
1182
1183 fn route_binary(&self, frame: &[u8]) -> Option<Vec<u8>> {
1187 let mut state = self.state.lock().expect("round lock");
1188 match state.round.route_binary(frame) {
1189 Ok(RoundInbound::Delta(body)) => Some(body),
1190 Ok(RoundInbound::RoundProgress) => {
1191 state.last_progress = Some(Instant::now());
1192 None
1193 }
1194 Ok(RoundInbound::Ignored) => None,
1195 Ok(RoundInbound::RoundComplete(bytes)) => {
1196 state.outcome = Some(Ok(bytes));
1197 self.ready.notify_all();
1198 None
1199 }
1200 Err(error) => {
1201 state.outcome = Some(Err(error));
1202 self.ready.notify_all();
1203 None
1204 }
1205 }
1206 }
1207
1208 fn fail_in_flight(&self, error: TransportError) {
1210 let mut state = self.state.lock().expect("round lock");
1211 state.lost = true;
1212 if state.round.in_flight() && state.outcome.is_none() {
1213 state.round.abort();
1214 state.outcome = Some(Err(error));
1215 self.ready.notify_all();
1216 }
1217 }
1218
1219 fn wait(
1222 &self,
1223 ceiling: Option<Instant>,
1224 idle: Duration,
1225 ) -> Result<Vec<u8>, TransportError> {
1226 let mut state = self.state.lock().expect("round lock");
1227 while state.outcome.is_none() {
1228 let (deadline, ceiling_binds) = round_expiry(
1229 state.last_progress.unwrap_or_else(Instant::now),
1230 idle,
1231 ceiling,
1232 );
1233 let now = Instant::now();
1234 if now >= deadline {
1235 state.round.abort();
1236 return Err(if ceiling_binds {
1237 timeout_error("sync round deadline exceeded", "round_deadline")
1238 } else {
1239 TransportError::new(
1240 "sync.transport_failed",
1241 "realtime sync round made no progress (§8.7)",
1242 )
1243 });
1244 }
1245 let (guard, _timeout) = self
1246 .ready
1247 .wait_timeout(state, deadline - now)
1248 .expect("round wait");
1249 state = guard;
1250 }
1251 state.outcome.take().expect("outcome present")
1252 }
1253 }
1254
1255 fn transfer_error(
1256 message: &'static str,
1257 cause_kind: &'static str,
1258 http_status: Option<u16>,
1259 ) -> TransportError {
1260 let mut details = serde_json::json!({ "causeKind": cause_kind });
1261 if let Some(status) = http_status {
1262 details["httpStatus"] = status.into();
1263 }
1264 let mut error = TransportError::new("transport.failed", message);
1265 error.details = Some(details);
1266 error
1267 }
1268
1269 fn timeout_error(message: &'static str, cause_kind: &'static str) -> TransportError {
1272 let mut error = TransportError::new("transport.timeout", message);
1273 error.details = Some(serde_json::json!({ "causeKind": cause_kind }));
1274 error
1275 }
1276
1277 fn redirect_error(message: &'static str, http_status: Option<u16>) -> TransportError {
1279 let mut details = serde_json::json!({ "causeKind": "redirect" });
1280 if let Some(status) = http_status {
1281 details["httpStatus"] = status.into();
1282 }
1283 let mut error = TransportError::new("transport.redirect", message);
1284 error.details = Some(details);
1285 error
1286 }
1287
1288 fn redirect_refused(resp: &ureq::http::Response<ureq::Body>) -> Result<(), TransportError> {
1293 let status = resp.status().as_u16();
1294 if (300..400).contains(&status) {
1295 return Err(redirect_error("redirect refused", Some(status)));
1296 }
1297 Ok(())
1298 }
1299
1300 fn is_timeout_io(error: &std::io::Error) -> bool {
1306 if error.kind() == std::io::ErrorKind::TimedOut {
1307 return true;
1308 }
1309 let Some(inner) = error.get_ref() else {
1310 return false;
1311 };
1312 let Some(ureq_error) = inner.downcast_ref::<ureq::Error>() else {
1313 return false;
1314 };
1315 match ureq_error {
1316 ureq::Error::Timeout(_) => true,
1317 ureq::Error::Io(nested) => is_timeout_io(nested),
1318 ureq::Error::Decompress(_, nested) => is_timeout_io(nested),
1319 _ => false,
1320 }
1321 }
1322
1323 fn http_err(message: &'static str, error: ureq::Error) -> TransportError {
1324 let (kind, status) = match error {
1325 ureq::Error::StatusCode(status) if (300..400).contains(&status) => {
1326 return redirect_error(message, Some(status))
1327 }
1328 ureq::Error::StatusCode(status) => ("status", Some(status)),
1329 ureq::Error::Timeout(_) => return timeout_error(message, "timeout"),
1330 ureq::Error::Io(error) => return io_err(message, error),
1331 ureq::Error::HostNotFound
1332 | ureq::Error::ConnectionFailed
1333 | ureq::Error::ConnectProxyFailed(_) => ("connect", None),
1334 ureq::Error::Tls(_)
1335 | ureq::Error::Rustls(_)
1336 | ureq::Error::Pem(_)
1337 | ureq::Error::TlsRequired => ("tls", None),
1338 ureq::Error::BadUri(_)
1339 | ureq::Error::Http(_)
1340 | ureq::Error::RequireHttpsOnly(_)
1341 | ureq::Error::InvalidProxyUrl => ("request", None),
1342 ureq::Error::Protocol(_) => ("protocol", None),
1343 ureq::Error::BodyExceedsLimit(_)
1344 | ureq::Error::BodyStalled
1345 | ureq::Error::Decompress(_, _) => ("body", None),
1346 ureq::Error::RedirectFailed | ureq::Error::TooManyRedirects => {
1347 return redirect_error(message, None)
1348 }
1349 _ => ("unknown", None),
1350 };
1351 transfer_error(message, kind, status)
1352 }
1353
1354 fn io_err(message: &'static str, error: std::io::Error) -> TransportError {
1355 if is_timeout_io(&error) {
1356 return timeout_error(message, "timeout");
1357 }
1358 let kind = match error.kind() {
1359 std::io::ErrorKind::ConnectionRefused
1360 | std::io::ErrorKind::ConnectionAborted
1361 | std::io::ErrorKind::ConnectionReset
1362 | std::io::ErrorKind::NotConnected => "connect",
1363 _ => "io",
1364 };
1365 transfer_error(message, kind, None)
1366 }
1367
1368 fn ws_err(message: &'static str, error: tungstenite::Error) -> TransportError {
1369 match error {
1370 tungstenite::Error::Io(error) => io_err(message, error),
1371 tungstenite::Error::Tls(_) => transfer_error(message, "tls", None),
1372 tungstenite::Error::Http(response) if response.status().is_redirection() => {
1375 redirect_error(message, Some(response.status().as_u16()))
1376 }
1377 tungstenite::Error::Http(response) => {
1378 transfer_error(message, "status", Some(response.status().as_u16()))
1379 }
1380 tungstenite::Error::Url(_) | tungstenite::Error::HttpFormat(_) => {
1381 transfer_error(message, "request", None)
1382 }
1383 _ => transfer_error(message, "protocol", None),
1384 }
1385 }
1386
1387 const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
1388
1389 fn remaining(deadline: Instant) -> io::Result<Duration> {
1390 deadline
1391 .checked_duration_since(Instant::now())
1392 .filter(|left| !left.is_zero())
1393 .ok_or_else(|| {
1394 io::Error::new(
1395 io::ErrorKind::TimedOut,
1396 "websocket establishment deadline exceeded",
1397 )
1398 })
1399 }
1400
1401 struct EstablishingStream {
1402 socket: TcpStream,
1403 deadline: Option<Instant>,
1404 }
1405
1406 impl EstablishingStream {
1407 fn timeout(&self) -> io::Result<Option<Duration>> {
1408 self.deadline.map(remaining).transpose()
1409 }
1410 fn established(&mut self) -> io::Result<TcpStream> {
1411 self.deadline = None;
1412 self.socket.set_read_timeout(None)?;
1413 self.socket.set_write_timeout(None)?;
1414 self.socket.try_clone()
1415 }
1416 }
1417
1418 fn establishment_io<T>(result: io::Result<T>, establishing: bool) -> io::Result<T> {
1419 result.map_err(|error| {
1420 if establishing
1421 && matches!(
1422 error.kind(),
1423 io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut
1424 )
1425 {
1426 io::Error::new(
1427 io::ErrorKind::TimedOut,
1428 "websocket establishment deadline exceeded",
1429 )
1430 } else {
1431 error
1432 }
1433 })
1434 }
1435
1436 impl Read for EstablishingStream {
1437 fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
1438 if self.deadline.is_some() {
1439 self.socket.set_read_timeout(self.timeout()?)?;
1440 }
1441 establishment_io(self.socket.read(bytes), self.deadline.is_some())
1442 }
1443 }
1444 impl Write for EstablishingStream {
1445 fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
1446 if self.deadline.is_some() {
1447 self.socket.set_write_timeout(self.timeout()?)?;
1448 }
1449 establishment_io(self.socket.write(bytes), self.deadline.is_some())
1450 }
1451 fn flush(&mut self) -> io::Result<()> {
1452 if self.deadline.is_some() {
1453 self.socket.set_write_timeout(self.timeout()?)?;
1454 }
1455 establishment_io(self.socket.flush(), self.deadline.is_some())
1456 }
1457 }
1458
1459 fn connect_websocket(
1460 request: tungstenite::handshake::client::Request,
1461 deadline: Instant,
1462 ) -> Result<tungstenite::WebSocket<MaybeTlsStream<EstablishingStream>>, TransportError> {
1463 let host = request
1464 .uri()
1465 .host()
1466 .ok_or_else(|| transfer_error("realtime URL invalid", "request", None))?;
1467 let host = host.trim_start_matches('[').trim_end_matches(']');
1468 let port = match request.uri().scheme_str() {
1469 Some("ws") => request.uri().port_u16().unwrap_or(80),
1470 Some("wss") => request.uri().port_u16().unwrap_or(443),
1471 _ => return Err(transfer_error("realtime URL invalid", "request", None)),
1472 };
1473 let host = host.to_owned();
1476 let (resolved, addresses) = mpsc::sync_channel(1);
1477 std::thread::Builder::new()
1478 .name("syncular-dns".to_owned())
1479 .spawn(move || {
1480 let result = (host.as_str(), port)
1481 .to_socket_addrs()
1482 .map(|items| items.collect::<Vec<_>>());
1483 let _ = resolved.send(result);
1484 })
1485 .map_err(|e| io_err("realtime DNS failed", e))?;
1486 let addresses = addresses
1487 .recv_timeout(remaining(deadline).map_err(|e| io_err("realtime DNS failed", e))?)
1488 .map_err(|e| match e {
1489 mpsc::RecvTimeoutError::Timeout => {
1490 timeout_error("realtime DNS timed out", "timeout")
1491 }
1492 mpsc::RecvTimeoutError::Disconnected => {
1493 transfer_error("realtime DNS failed", "io", None)
1494 }
1495 })?
1496 .map_err(|e| io_err("realtime DNS failed", e))?;
1497 let mut last_error = io::Error::new(io::ErrorKind::NotFound, "DNS returned no addresses");
1498 let mut socket = None;
1499 for address in addresses {
1500 match TcpStream::connect_timeout(
1501 &address,
1502 remaining(deadline).map_err(|e| io_err("realtime TCP connect failed", e))?,
1503 ) {
1504 Ok(stream) => {
1505 socket = Some(stream);
1506 break;
1507 }
1508 Err(error) => last_error = error,
1509 }
1510 }
1511 let socket = socket.ok_or_else(|| io_err("realtime TCP connect failed", last_error))?;
1512 socket
1513 .set_nodelay(true)
1514 .map_err(|e| io_err("realtime TCP connect failed", e))?;
1515 let stream = EstablishingStream {
1516 socket,
1517 deadline: Some(deadline),
1518 };
1519 let (ws, _) = tungstenite::client_tls_with_config(request, stream, None, None).map_err(
1522 |e| match e {
1523 tungstenite::HandshakeError::Failure(error) => {
1524 ws_err("realtime connection failed", error)
1525 }
1526 tungstenite::HandshakeError::Interrupted(_) => {
1529 transfer_error("realtime connection failed", "protocol", None)
1530 }
1531 },
1532 )?;
1533 remaining(deadline).map_err(|e| io_err("realtime connection failed", e))?;
1534 Ok(ws)
1535 }
1536
1537 #[cfg(test)]
1538 mod establishment_tests {
1539 use super::*;
1540 use std::net::{Shutdown, TcpListener};
1541 use tungstenite::client::IntoClientRequest;
1542
1543 fn withheld_response(tls: bool, old: bool) {
1546 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1547 let url = format!(
1548 "{}://{}/realtime",
1549 if tls { "wss" } else { "ws" },
1550 listener.local_addr().unwrap()
1551 );
1552 let (accepted_tx, accepted_rx) = mpsc::channel();
1553 let (release_tx, release_rx) = mpsc::channel();
1554 let peer = std::thread::spawn(move || {
1555 let (mut stream, _) = listener.accept().unwrap();
1556 stream
1557 .set_read_timeout(Some(Duration::from_secs(2)))
1558 .unwrap();
1559 let mut bytes = [0; 4096];
1560 assert!(stream.read(&mut bytes).unwrap() > 0);
1561 accepted_tx.send(()).unwrap();
1562 release_rx.recv_timeout(Duration::from_secs(3)).unwrap();
1563 let _ = stream.shutdown(Shutdown::Both);
1564 });
1565 let (done_tx, done_rx) = mpsc::channel();
1566 let worker = std::thread::spawn(move || {
1567 let result = if old {
1568 tungstenite::connect(url)
1569 .map(|_| ())
1570 .map_err(|e| e.to_string())
1571 } else {
1572 connect_websocket(
1573 url.into_client_request().unwrap(),
1574 Instant::now() + Duration::from_millis(250),
1575 )
1576 .map(|_| ())
1577 .map_err(|e| format!("{}: {}", e.code, e.message))
1578 };
1579 done_tx.send(result).unwrap();
1580 });
1581 accepted_rx.recv_timeout(Duration::from_secs(2)).unwrap();
1582 if old {
1583 assert!(
1584 matches!(
1585 done_rx.recv_timeout(Duration::from_millis(500)),
1586 Err(mpsc::RecvTimeoutError::Timeout)
1587 ),
1588 "original connect unexpectedly had a deadline"
1589 );
1590 release_tx.send(()).unwrap();
1591 assert!(done_rx
1592 .recv_timeout(Duration::from_secs(2))
1593 .unwrap()
1594 .is_err());
1595 } else {
1596 let error = done_rx
1597 .recv_timeout(Duration::from_secs(2))
1598 .expect("establishment remained blocked")
1599 .unwrap_err();
1600 assert_eq!(error, "transport.timeout: realtime connection failed");
1601 eprintln!(
1602 "withheld {} produces {error}",
1603 if tls { "TLS" } else { "upgrade" }
1604 );
1605 release_tx.send(()).unwrap();
1606 }
1607 worker.join().unwrap();
1608 peer.join().unwrap();
1609 }
1610
1611 #[test]
1612 fn original_connect_blocks_on_withheld_upgrade_and_tls() {
1613 withheld_response(false, true);
1614 withheld_response(true, true);
1615 }
1616 #[test]
1617 fn establishment_deadline_errors_on_withheld_upgrade_and_tls() {
1618 withheld_response(false, false);
1619 withheld_response(true, false);
1620 }
1621
1622 #[test]
1623 fn slow_upgrade_bytes_cannot_renew_the_absolute_deadline() {
1624 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1625 let url = format!("ws://{}", listener.local_addr().unwrap());
1626 let peer = std::thread::spawn(move || {
1627 let (mut stream, _) = listener.accept().unwrap();
1628 let mut bytes = [0; 4096];
1629 let _ = stream.read(&mut bytes).unwrap();
1630 for byte in b"HTTP/1.1 101 Switching Protocols\r\n" {
1631 if stream.write_all(&[*byte]).is_err() {
1632 break;
1633 }
1634 std::thread::sleep(Duration::from_millis(40));
1635 }
1636 });
1637 let started = Instant::now();
1638 assert!(connect_websocket(
1639 url.into_client_request().unwrap(),
1640 started + Duration::from_millis(250)
1641 )
1642 .is_err());
1643 assert!(started.elapsed() < Duration::from_secs(1));
1644 peer.join().unwrap();
1645 }
1646
1647 #[test]
1648 fn saturated_tcp_accept_queue_returns_an_observable_error() {
1649 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1652 let address = listener.local_addr().unwrap();
1653 let mut queued = Vec::new();
1654 loop {
1655 match TcpStream::connect_timeout(&address, Duration::from_millis(50)) {
1656 Ok(stream) => queued.push(stream),
1657 Err(error) => {
1658 assert!(
1659 matches!(
1660 error.kind(),
1661 io::ErrorKind::TimedOut
1662 | io::ErrorKind::WouldBlock
1663 | io::ErrorKind::ConnectionRefused
1664 | io::ErrorKind::ConnectionReset
1665 ),
1666 "{error:?}"
1667 );
1668 break;
1669 }
1670 }
1671 assert!(queued.len() < 4096, "listener queue did not saturate");
1672 }
1673 let started = Instant::now();
1674 let url = format!("ws://{address}");
1675 let error = match connect_websocket(
1676 url.into_client_request().unwrap(),
1677 started + Duration::from_millis(250),
1678 ) {
1679 Err(error) => error,
1680 Ok(_) => panic!("full accept queue unexpectedly connected"),
1681 };
1682 assert!(
1683 (error.code == "transport.timeout" || error.code == "transport.failed")
1684 && (error.message == "realtime TCP connect failed"
1685 || error.message == "realtime connection failed"),
1686 "{}: {}",
1687 error.code,
1688 error.message
1689 );
1690 assert!(started.elapsed() < Duration::from_secs(2));
1691 }
1692 }
1693
1694 fn segment_error(mut error: TransportError) -> TransportError {
1698 if error.code == "transport.failed" {
1699 error.code = "sync.transport_failed".into();
1700 error.message = "segment transfer failed".into();
1701 }
1702 error
1703 }
1704
1705 fn is_would_block(e: &tungstenite::Error) -> bool {
1707 matches!(
1708 e,
1709 tungstenite::Error::Io(io) if matches!(
1710 io.kind(),
1711 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
1712 )
1713 )
1714 }
1715
1716 pub struct NativeTransport {
1717 base_url: String,
1718 ws_url: String,
1719 headers: Vec<(String, String)>,
1721 base_url_has_userinfo: bool,
1725 agent: ureq::Agent,
1726 policy: HostTransportPolicy,
1727 round_deadline_at: Option<std::time::Instant>,
1731 pub signed_urls: bool,
1732 pub inbound: Arc<InboundBuffer>,
1733 outgoing: Option<Box<SocketSender>>,
1735 reader: Option<JoinHandle<()>>,
1736 reader_stop: Arc<AtomicBool>,
1737 round: Arc<RoundChannel>,
1739 }
1740
1741 #[derive(Debug)]
1748 struct IdleConnector(Option<Duration>);
1749
1750 impl Connector<Box<dyn UreqTransport>> for IdleConnector {
1751 type Out = Box<dyn UreqTransport>;
1752
1753 fn connect(
1754 &self,
1755 _: &ConnectionDetails,
1756 chained: Option<Box<dyn UreqTransport>>,
1757 ) -> Result<Option<Self::Out>, ureq::Error> {
1758 Ok(chained.map(|inner| match self.0 {
1759 Some(idle) => IdleTransport { inner, idle }.boxed(),
1760 None => inner,
1761 }))
1762 }
1763 }
1764
1765 #[derive(Debug)]
1766 struct IdleTransport {
1767 inner: Box<dyn UreqTransport>,
1768 idle: Duration,
1769 }
1770
1771 fn idle_bound(mut timeout: NextTimeout, idle: Duration) -> NextTimeout {
1776 let idle = UreqDuration::Exact(idle);
1777 if timeout.after.is_not_happening() || timeout.after.as_nanos() > idle.as_nanos() {
1778 timeout.after = idle;
1779 timeout.reason = UreqTimeout::RecvBody;
1780 }
1781 timeout
1782 }
1783
1784 impl UreqTransport for IdleTransport {
1785 fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers {
1786 self.inner.buffers()
1787 }
1788 fn transmit_output(
1789 &mut self,
1790 amount: usize,
1791 timeout: NextTimeout,
1792 ) -> Result<(), ureq::Error> {
1793 self.inner
1794 .transmit_output(amount, idle_bound(timeout, self.idle))
1795 }
1796 fn await_input(&mut self, timeout: NextTimeout) -> Result<bool, ureq::Error> {
1797 self.inner.await_input(idle_bound(timeout, self.idle))
1798 }
1799 fn is_open(&mut self) -> bool {
1800 self.inner.is_open()
1801 }
1802 fn is_tls(&self) -> bool {
1803 self.inner.is_tls()
1804 }
1805 }
1806
1807 fn build_agent(policy: &HostTransportPolicy) -> ureq::Agent {
1808 let config = ureq::Agent::config_builder()
1809 .timeout_resolve(policy.request_timeout)
1810 .timeout_connect(policy.request_timeout)
1811 .timeout_send_request(policy.request_timeout)
1812 .timeout_recv_response(policy.request_timeout)
1813 .max_redirects(match policy.redirects {
1814 RedirectPolicy::Deny => 0,
1815 RedirectPolicy::Follow => 10,
1816 })
1817 .max_redirects_will_error(matches!(policy.redirects, RedirectPolicy::Follow))
1818 .redirect_auth_headers(ureq::config::RedirectAuthHeaders::Never)
1819 .http_status_as_error(false)
1824 .build();
1825 ureq::Agent::with_parts(
1826 config,
1827 DefaultConnector::new().chain(IdleConnector(policy.request_timeout)),
1828 DefaultResolver::default(),
1829 )
1830 }
1831
1832 fn derive_ws_url(base_url: &str) -> String {
1833 let ws = if let Some(rest) = base_url.strip_prefix("https://") {
1836 format!("wss://{rest}")
1837 } else if let Some(rest) = base_url.strip_prefix("http://") {
1838 format!("ws://{rest}")
1839 } else {
1840 base_url.to_owned()
1841 };
1842 let trimmed = ws.trim_end_matches('/');
1843 format!("{trimmed}/realtime")
1844 }
1845
1846 impl NativeTransport {
1847 pub fn fork_round(&self) -> Self {
1848 Self {
1849 base_url: self.base_url.clone(),
1850 ws_url: self.ws_url.clone(),
1851 headers: self.headers.clone(),
1852 base_url_has_userinfo: self.base_url_has_userinfo,
1853 agent: self.agent.clone(),
1854 policy: self.policy.clone(),
1855 round_deadline_at: None,
1856 signed_urls: self.signed_urls,
1857 inbound: self.inbound.clone(),
1858 outgoing: self.outgoing.clone(),
1859 reader: None,
1860 reader_stop: self.reader_stop.clone(),
1861 round: self.round.clone(),
1862 }
1863 }
1864
1865 pub fn new(
1866 base_url: &str,
1867 config: &serde_json::Value,
1868 notify: Option<Arc<dyn Fn() + Send + Sync>>,
1869 ) -> Result<Self, String> {
1870 let policy = HostTransportPolicy::from_config(config)?;
1871 Self::with_policy(base_url, config, policy, notify)
1872 }
1873
1874 pub fn with_policy(
1875 base_url: &str,
1876 config: &serde_json::Value,
1877 policy: HostTransportPolicy,
1878 notify: Option<Arc<dyn Fn() + Send + Sync>>,
1879 ) -> Result<Self, String> {
1880 policy.validate()?;
1881 let mut headers = Vec::new();
1882 if let Some(map) = config.get("headers").and_then(|v| v.as_object()) {
1883 for (k, v) in map {
1884 if let Some(s) = v.as_str() {
1885 headers.push((k.clone(), s.to_owned()));
1886 }
1887 }
1888 }
1889 let ws_url = config
1890 .get("wsUrl")
1891 .and_then(|v| v.as_str())
1892 .map(str::to_owned)
1893 .unwrap_or_else(|| derive_ws_url(base_url));
1894 let base_url_has_userinfo = url::Url::parse(base_url)
1897 .map(|url| !url.username().is_empty() || url.password().is_some())
1898 .unwrap_or(false);
1899 Ok(NativeTransport {
1900 base_url: base_url.trim_end_matches('/').to_owned(),
1901 ws_url,
1902 headers,
1903 base_url_has_userinfo,
1904 agent: build_agent(&policy),
1905 policy,
1906 round_deadline_at: None,
1907 signed_urls: false,
1908 inbound: Arc::new(match notify {
1909 Some(notify) => InboundBuffer::with_notify(notify),
1910 None => InboundBuffer::default(),
1911 }),
1912 outgoing: None,
1913 reader: None,
1914 reader_stop: Arc::new(AtomicBool::new(false)),
1915 round: Arc::new(RoundChannel::default()),
1916 })
1917 }
1918
1919 pub fn set_policy(&mut self, policy: HostTransportPolicy) -> Result<(), String> {
1923 policy.validate()?;
1924 self.agent = build_agent(&policy);
1925 self.policy = policy;
1926 Ok(())
1927 }
1928
1929 pub fn round_deadline(&self) -> Option<Duration> {
1930 self.policy.round_deadline
1931 }
1932
1933 pub fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
1934 self.round_deadline_at = deadline;
1935 }
1936
1937 fn round_remaining(&self) -> Result<Option<Duration>, TransportError> {
1942 match self.round_deadline_at {
1943 Some(deadline) => deadline
1944 .checked_duration_since(std::time::Instant::now())
1945 .map(Some)
1946 .ok_or_else(|| timeout_error("sync round deadline exceeded", "round_deadline")),
1947 None => Ok(None),
1948 }
1949 }
1950
1951 fn credentialed(&self) -> bool {
1954 !self.headers.is_empty() || self.base_url_has_userinfo
1955 }
1956
1957 fn finish_request<B>(
1963 &self,
1964 req: ureq::RequestBuilder<B>,
1965 credentialed: bool,
1966 ) -> Result<ureq::RequestBuilder<B>, TransportError> {
1967 let timeout = self.round_remaining()?;
1968 let follow = self.policy.redirects == RedirectPolicy::Follow && !credentialed;
1969 Ok(req
1970 .config()
1971 .timeout_global(timeout)
1972 .max_redirects(if follow { 10 } else { 0 })
1973 .max_redirects_will_error(true)
1974 .build())
1975 }
1976
1977 fn check_request_bytes(&self, len: u64) -> Result<(), TransportError> {
1979 match self.policy.max_request_bytes {
1980 Some(max) if len > max => {
1981 let mut error = TransportError::new(
1982 "transport.request_too_large",
1983 "request body exceeds the configured transport limit",
1984 );
1985 error.details = Some(serde_json::json!({ "limit": max, "size": len }));
1986 Err(error)
1987 }
1988 _ => Ok(()),
1989 }
1990 }
1991
1992 fn post_sync(&self, path: &str, body: &[u8]) -> Result<Vec<u8>, TransportError> {
1993 self.check_request_bytes(body.len() as u64)?;
1994 let credentialed = self.credentialed();
1995 let url = format!("{}{}", self.base_url, path);
1996 let mut req = self
1999 .agent
2000 .post(&url)
2001 .header("content-type", "application/vnd.syncular.sync.v2");
2002 for (k, v) in &self.headers {
2003 req = req.header(k.as_str(), v.as_str());
2004 }
2005 let req = self.finish_request(req, credentialed)?;
2006 let resp = req
2007 .send(body)
2008 .map_err(|e| http_err("sync request failed", e))?;
2009 read_body(resp, None, self.policy.max_response_bytes, None)
2010 }
2011
2012 fn post_operation(&self, body: &[u8]) -> Result<Vec<u8>, TransportError> {
2013 self.check_request_bytes(body.len() as u64)?;
2014 let credentialed = self.credentialed();
2015 let url = format!("{}/operations", self.base_url);
2016 let mut req = self.agent.post(&url).header(
2017 "content-type",
2018 "application/vnd.syncular.operations.v1+json",
2019 );
2020 for (key, value) in &self.headers {
2021 req = req.header(key.as_str(), value.as_str());
2022 }
2023 let req = self.finish_request(req, credentialed)?;
2024 let response = req
2025 .send(body)
2026 .map_err(|error| http_err("remote operation request failed", error))?;
2027 read_body(response, None, self.policy.max_response_bytes, None)
2028 }
2029
2030 pub fn set_headers(&mut self, headers: Vec<(String, String)>) {
2031 self.headers = headers;
2032 }
2033
2034 pub fn shutdown(&mut self) {
2035 self.reader_stop.store(true, Ordering::SeqCst);
2036 self.round.fail_in_flight(TransportError::new(
2039 "sync.transport_failed",
2040 "realtime disconnected mid-round (§8.7)",
2041 ));
2042 if let Some(outgoing) = &self.outgoing {
2043 let _ = outgoing.poller.notify();
2044 }
2045 if let Some(handle) = self.reader.take() {
2046 let _ = handle.join();
2047 }
2048 self.outgoing = None;
2049 }
2050
2051 fn send_message(&mut self, message: Message) -> Result<(), TransportError> {
2052 let (wait, round_bound) = match self.round_deadline_at {
2057 Some(deadline) => {
2058 match deadline.checked_duration_since(std::time::Instant::now()) {
2059 Some(remaining) if remaining < ROUND_IDLE_TIMEOUT => (remaining, true),
2060 Some(_) => (ROUND_IDLE_TIMEOUT, false),
2061 None => {
2064 return Err(timeout_error("realtime send timed out", "round_deadline"));
2065 }
2066 }
2067 }
2068 None => (ROUND_IDLE_TIMEOUT, false),
2069 };
2070 let result = (|| {
2071 let Some(outgoing) = &self.outgoing else {
2072 return Err(TransportError::new(
2073 "transport.failed",
2074 "realtime not connected",
2075 ));
2076 };
2077 let (completed, result) = mpsc::channel();
2078 outgoing
2079 .queue
2080 .try_send(Outgoing { message, completed })
2081 .map_err(|_| {
2082 TransportError::new("transport.failed", "realtime send queue unavailable")
2083 })?;
2084 outgoing
2085 .poller
2086 .notify()
2087 .map_err(|e| io_err("realtime wake failed", e))?;
2088 result.recv_timeout(wait).map_err(|_| {
2089 if round_bound {
2090 timeout_error("realtime send timed out", "round_deadline")
2091 } else {
2092 TransportError::new("transport.failed", "realtime send did not complete")
2093 }
2094 })?
2095 })();
2096 if let Err(error) = &result {
2099 self.round.fail_in_flight(error.clone());
2103 self.shutdown();
2104 self.inbound.push(Inbound::Disconnected(error.clone()));
2107 }
2108 result
2109 }
2110 }
2111
2112 fn http_status_body_error(status: u16, body: &[u8]) -> TransportError {
2116 let mut error = TransportError::new("transport.failed", "HTTP request failed");
2117 let mut details = serde_json::Map::new();
2118 if let Ok(serde_json::Value::Object(parsed)) =
2119 serde_json::from_slice::<serde_json::Value>(body)
2120 {
2121 if let Some(code) = parsed.get("code").and_then(serde_json::Value::as_str) {
2122 if !code.is_empty() {
2123 error.code = code.to_owned();
2124 }
2125 }
2126 if let Some(message) = parsed.get("message").and_then(serde_json::Value::as_str) {
2127 error.message = message.to_owned();
2128 }
2129 for key in ["category", "recommendedAction"] {
2130 if let Some(value) = parsed.get(key).and_then(serde_json::Value::as_str) {
2131 details.insert(key.to_owned(), serde_json::Value::from(value));
2132 }
2133 }
2134 if let Some(retryable) = parsed.get("retryable").and_then(serde_json::Value::as_bool) {
2135 details.insert("retryable".to_owned(), serde_json::Value::from(retryable));
2136 }
2137 if let Some(serde_json::Value::Object(inner)) = parsed.get("details") {
2138 for (key, value) in inner {
2139 details.insert(key.clone(), value.clone());
2140 }
2141 }
2142 }
2143 details.insert("causeKind".to_owned(), serde_json::Value::from("status"));
2144 details.insert("httpStatus".to_owned(), serde_json::Value::from(status));
2145 error.details = Some(serde_json::Value::Object(details));
2146 error
2147 }
2148
2149 fn read_body(
2154 resp: ureq::http::Response<ureq::Body>,
2155 mut progress: Option<&mut dyn FnMut(u64)>,
2156 max: Option<u64>,
2157 stop: Option<u64>,
2158 ) -> Result<Vec<u8>, TransportError> {
2159 redirect_refused(&resp)?;
2160 use std::io::Read;
2161 let status = resp.status().as_u16();
2162 let mut reader = resp.into_body().into_with_config().reader();
2163 let mut bytes = Vec::new();
2164 let mut chunk = [0u8; 64 * 1024];
2165 let mut reported = 0;
2166 loop {
2167 let n = reader.read(&mut chunk).map_err(|error| {
2168 if is_timeout_io(&error) {
2169 timeout_error("response body read times out", "timeout")
2170 } else {
2171 transfer_error("response body read failed", "body", Some(status))
2172 }
2173 })?;
2174 if n == 0 {
2175 break;
2176 }
2177 if let Some(max) = max {
2180 if bytes.len() as u64 + n as u64 > max {
2181 return Err(response_too_large(max));
2182 }
2183 }
2184 bytes.extend_from_slice(&chunk[..n]);
2185 if stop.is_some_and(|stop| bytes.len() as u64 > stop) {
2186 break;
2187 }
2188 if bytes.len() - reported >= 64 * 1024 {
2189 if let Some(callback) = progress.as_mut() {
2190 callback(bytes.len() as u64);
2191 }
2192 reported = bytes.len();
2193 }
2194 }
2195 if bytes.len() != reported {
2196 if let Some(callback) = progress {
2197 callback(bytes.len() as u64);
2198 }
2199 }
2200 if status >= 400 {
2201 return Err(http_status_body_error(status, &bytes));
2202 }
2203 Ok(bytes)
2204 }
2205
2206 fn response_too_large(max: u64) -> TransportError {
2207 let mut error = TransportError::new(
2208 "transport.response_too_large",
2209 "response body exceeds the configured transport limit",
2210 );
2211 error.details = Some(serde_json::json!({ "limit": max }));
2212 error
2213 }
2214
2215 impl Transport for NativeTransport {
2216 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2217 self.post_sync("/sync", request)
2218 }
2219
2220 fn remote_operation(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2221 self.post_operation(request)
2222 }
2223
2224 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
2225 if self.outgoing.is_none() {
2235 return self.post_sync("/sync", request);
2236 }
2237 self.round_remaining()?;
2238 let framed = self.round.begin(request)?;
2239 let (tag, body) = framed.split_first().expect("framed request carries a tag");
2240 for chunk in body.chunks(REQUEST_CHUNK_BYTES) {
2241 let mut message = Vec::with_capacity(chunk.len() + 1);
2242 message.push(*tag);
2243 message.extend_from_slice(chunk);
2244 if let Err(error) = self.send_message(Message::Binary(message.into())) {
2245 self.round.fail_in_flight(error);
2246 break;
2247 }
2248 self.round.progressed();
2249 }
2250 let outcome = self.round.wait(self.round_deadline_at, ROUND_IDLE_TIMEOUT);
2251 if let Err(error) = &outcome {
2256 if !self.round.lost() {
2257 self.shutdown();
2258 self.inbound.push(Inbound::Disconnected(error.clone()));
2259 }
2260 }
2261 outcome
2262 }
2263
2264 fn download_segment(
2265 &mut self,
2266 request: &SegmentRequest,
2267 on_progress: &mut dyn FnMut(u64),
2268 ) -> Result<Vec<u8>, TransportError> {
2269 let url = format!("{}/segments/{}", self.base_url, request.segment_id);
2276 let credentialed = self.credentialed();
2277 let mut req = self
2278 .agent
2279 .get(&url)
2280 .header("x-syncular-scopes", &request.requested_scopes_json);
2281 for (k, v) in &self.headers {
2282 req = req.header(k.as_str(), v.as_str());
2283 }
2284 let req = self.finish_request(req, credentialed)?;
2285 let resp = req
2286 .call()
2287 .map_err(|error| segment_error(http_err("segment request failed", error)))?;
2288 read_body(
2289 resp,
2290 Some(on_progress),
2291 self.policy.max_response_bytes,
2292 Some(request.byte_length),
2293 )
2294 .map_err(segment_error)
2295 }
2296
2297 fn supports_url_fetch(&self) -> bool {
2298 self.signed_urls
2299 }
2300
2301 fn download_segments(
2302 &mut self,
2303 fetches: &[SegmentFetch],
2304 on_progress: &mut (dyn FnMut(usize, u64) + Send),
2305 ) -> Vec<Result<Vec<u8>, TransportError>> {
2306 let workers = (0..fetches.len().min(4))
2315 .map(|_| {
2316 let mut worker = self.fork_round();
2317 worker.round_deadline_at = self.round_deadline_at;
2318 worker
2319 })
2320 .collect::<Vec<_>>();
2321 let slots = fetches.iter().map(|_| Mutex::new(None)).collect::<Vec<_>>();
2322 let progress = Mutex::new(on_progress);
2323 let next = std::sync::atomic::AtomicUsize::new(0);
2324 std::thread::scope(|scope| {
2325 for worker in workers {
2326 let (slots, progress, next) = (&slots, &progress, &next);
2327 scope.spawn(move || {
2328 let mut worker = worker;
2329 loop {
2330 let index = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2331 let Some(fetch) = fetches.get(index) else {
2332 break;
2333 };
2334 let mut report = |bytes: u64| {
2335 (*progress.lock().expect("segment progress lock"))(index, bytes);
2336 };
2337 let result = match fetch {
2338 SegmentFetch::Direct(request) => {
2339 worker.download_segment(request, &mut report)
2340 }
2341 SegmentFetch::Url { url, byte_length } => {
2342 worker.fetch_url(url, *byte_length, &mut report)
2343 }
2344 };
2345 *slots[index].lock().expect("segment result slot") = Some(result);
2346 }
2347 });
2348 }
2349 });
2350 slots
2351 .into_iter()
2352 .map(|slot| slot.into_inner().expect("segment result slot"))
2353 .map(|result| result.expect("every fetch is attempted"))
2354 .collect()
2355 }
2356
2357 fn fetch_url(
2358 &mut self,
2359 url: &str,
2360 byte_length: u64,
2361 on_progress: &mut dyn FnMut(u64),
2362 ) -> Result<Vec<u8>, TransportError> {
2363 let req = self.finish_request(self.agent.get(url), true)?;
2365 let resp = req
2366 .call()
2367 .map_err(|error| segment_error(http_err("segment request failed", error)))?;
2368 read_body(
2369 resp,
2370 Some(on_progress),
2371 self.policy.max_response_bytes,
2372 Some(byte_length),
2373 )
2374 .map_err(segment_error)
2375 }
2376
2377 fn blob_upload(
2378 &mut self,
2379 blob_id: &str,
2380 bytes: &[u8],
2381 media_type: Option<&str>,
2382 ) -> Result<(), TransportError> {
2383 self.check_request_bytes(bytes.len() as u64)?;
2386 let credentialed = self.credentialed();
2387 let url = format!("{}/blobs/{}", self.base_url, blob_id);
2388 let mut req = self.agent.put(&url).header(
2389 "content-type",
2390 media_type.unwrap_or("application/octet-stream"),
2391 );
2392 for (k, v) in &self.headers {
2393 req = req.header(k.as_str(), v.as_str());
2394 }
2395 let req = self.finish_request(req, credentialed)?;
2396 let resp = req
2397 .send(bytes)
2398 .map_err(|e| http_err("blob upload failed", e))?;
2399 read_body(resp, None, self.policy.max_response_bytes, None)?;
2400 Ok(())
2401 }
2402
2403 fn blob_download(
2404 &mut self,
2405 blob_id: &str,
2406 byte_length: Option<u64>,
2407 ) -> Result<BlobDownload, TransportError> {
2408 let credentialed = self.credentialed();
2411 let url = format!("{}/blobs/{}", self.base_url, blob_id);
2412 let mut req = self.agent.get(&url);
2413 for (k, v) in &self.headers {
2414 req = req.header(k.as_str(), v.as_str());
2415 }
2416 let req = self.finish_request(req, credentialed)?;
2417 let resp = req.call().map_err(|cause| {
2418 let status = match &cause {
2419 ureq::Error::StatusCode(status) => Some(*status),
2420 _ => None,
2421 };
2422 let mut error = http_err("blob download failed", cause);
2423 match status {
2424 Some(404) => {
2425 error.code = "blob.not_found".into();
2426 error.message = "blob not found".into();
2427 }
2428 Some(403) => {
2429 error.code = "blob.forbidden".into();
2430 error.message = "blob download forbidden".into();
2431 }
2432 Some(401) => {
2433 error.code = "sync.auth_required".into();
2434 error.message = "host authentication required".into();
2435 }
2436 _ => {}
2437 }
2438 error
2439 })?;
2440 let is_json = resp
2443 .headers()
2444 .get(ureq::http::header::CONTENT_TYPE)
2445 .and_then(|value| value.to_str().ok())
2446 .is_some_and(|value| value.contains("application/json"));
2447 let stop = byte_length.filter(|_| !is_json);
2450 let body = read_body(resp, None, self.policy.max_response_bytes, stop).map_err(
2451 |mut error| {
2452 let status = error
2456 .details
2457 .as_ref()
2458 .filter(|details| {
2459 details.get("causeKind").and_then(serde_json::Value::as_str)
2460 == Some("status")
2461 })
2462 .and_then(|details| details.get("httpStatus"))
2463 .and_then(serde_json::Value::as_u64);
2464 match status {
2465 Some(404) => {
2466 error.code = "blob.not_found".into();
2467 error.message = "blob not found".into();
2468 }
2469 Some(403) => {
2470 error.code = "blob.forbidden".into();
2471 error.message = "blob download forbidden".into();
2472 }
2473 Some(401) => {
2474 error.code = "sync.auth_required".into();
2475 error.message = "host authentication required".into();
2476 }
2477 Some(_) => {
2478 error.code = "transport.failed".into();
2479 error.message = "blob download failed".into();
2480 }
2481 None => {}
2482 }
2483 error
2484 },
2485 )?;
2486 if is_json {
2487 if let Ok(parsed) = serde_json::from_slice::<serde_json::Value>(&body) {
2488 if let Some(u) = parsed.get("url").and_then(|v| v.as_str()) {
2489 return Ok(BlobDownload::Url {
2490 url: u.to_owned(),
2491 url_expires_at_ms: parsed
2492 .get("urlExpiresAtMs")
2493 .and_then(|v| v.as_i64()),
2494 });
2495 }
2496 }
2497 }
2498 Ok(BlobDownload::Bytes(body))
2499 }
2500
2501 fn fetch_blob_url(
2502 &mut self,
2503 url: &str,
2504 byte_length: Option<u64>,
2505 ) -> Result<Vec<u8>, TransportError> {
2506 let req = self.finish_request(self.agent.get(url), true)?;
2509 let resp = req.call().map_err(|e| http_err("URL fetch failed", e))?;
2510 read_body(resp, None, self.policy.max_response_bytes, byte_length)
2511 }
2512
2513 fn blob_upload_grant(
2514 &mut self,
2515 blob_id: &str,
2516 byte_length: u64,
2517 media_type: Option<&str>,
2518 ) -> Result<BlobUploadGrant, TransportError> {
2519 let url = format!("{}/blobs/{}/upload-grant", self.base_url, blob_id);
2520 let body = serde_json::json!({
2521 "byteLength": byte_length,
2522 "mediaType": media_type,
2523 })
2524 .to_string();
2525 self.check_request_bytes(body.len() as u64)?;
2526 let credentialed = self.credentialed();
2527 let mut req = self
2528 .agent
2529 .post(&url)
2530 .header("content-type", "application/json");
2531 for (k, v) in &self.headers {
2532 req = req.header(k.as_str(), v.as_str());
2533 }
2534 let req = self.finish_request(req, credentialed)?;
2535 let resp = req
2536 .send(body)
2537 .map_err(|e| http_err("blob upload grant request failed", e))?;
2538 let grant_body = read_body(resp, None, self.policy.max_response_bytes, None)?;
2539 let parsed: serde_json::Value = serde_json::from_slice(&grant_body)
2540 .map_err(|_| transfer_error("blob upload grant decode failed", "decode", None))?;
2541 if let Some(u) = parsed.get("url").and_then(|v| v.as_str()) {
2542 return Ok(BlobUploadGrant::Url {
2543 url: u.to_owned(),
2544 url_expires_at_ms: parsed.get("urlExpiresAtMs").and_then(|v| v.as_i64()),
2545 });
2546 }
2547 if parsed.get("present").and_then(|v| v.as_bool()) == Some(true) {
2548 return Ok(BlobUploadGrant::Present);
2549 }
2550 Ok(BlobUploadGrant::None)
2551 }
2552
2553 fn blob_put_url(
2554 &mut self,
2555 blob_id: &str,
2556 url: &str,
2557 bytes: &[u8],
2558 media_type: Option<&str>,
2559 ) -> Result<(), TransportError> {
2560 let digest = blob_id
2565 .strip_prefix("sha256:")
2566 .filter(|hex| {
2567 hex.len() == 64
2568 && hex
2569 .bytes()
2570 .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
2571 })
2572 .map(|hex| {
2573 (0..64)
2574 .step_by(2)
2575 .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).expect("hex digits"))
2576 .collect::<Vec<u8>>()
2577 })
2578 .ok_or_else(|| {
2579 TransportError::new(
2580 "sync.invalid_request",
2581 "the blob id is not a sha256 blob id (§5.9.1)",
2582 )
2583 })?;
2584 self.check_request_bytes(bytes.len() as u64)?;
2585 let req = self.finish_request(
2586 self.agent
2587 .put(url)
2588 .header(
2589 "content-type",
2590 media_type.unwrap_or("application/octet-stream"),
2591 )
2592 .header("x-amz-checksum-sha256", ssp2::util::base64(&digest)),
2593 true,
2594 )?;
2595 let resp = req
2596 .send(bytes)
2597 .map_err(|e| http_err("signed blob upload failed", e))?;
2598 read_body(resp, None, self.policy.max_response_bytes, None)?;
2599 Ok(())
2600 }
2601
2602 fn realtime_connect(&mut self, client_id: &str) -> Result<(), TransportError> {
2603 if self.outgoing.is_some() {
2604 if self.reader_stop.load(Ordering::SeqCst)
2605 || self
2606 .reader
2607 .as_ref()
2608 .is_some_and(|reader| reader.is_finished())
2609 {
2610 self.shutdown();
2611 } else {
2612 return Ok(());
2613 }
2614 }
2615 use tungstenite::client::IntoClientRequest;
2621 let mut url = url::Url::parse(&self.ws_url)
2622 .map_err(|_| transfer_error("realtime URL invalid", "request", None))?;
2623 let retained_query: Vec<(String, String)> = url
2624 .query_pairs()
2625 .filter(|(key, _)| key != "clientId")
2626 .map(|(key, value)| (key.into_owned(), value.into_owned()))
2627 .collect();
2628 url.set_query(None);
2629 url.query_pairs_mut()
2630 .extend_pairs(retained_query)
2631 .append_pair("clientId", client_id);
2632 let mut request = url
2633 .as_str()
2634 .into_client_request()
2635 .map_err(|error| ws_err("realtime request failed", error))?;
2636 {
2637 let out = request.headers_mut();
2638 for (k, v) in &self.headers {
2639 if let (Ok(name), Ok(value)) = (
2640 tungstenite::http::HeaderName::try_from(k.as_str()),
2641 tungstenite::http::HeaderValue::try_from(v.as_str()),
2642 ) {
2643 out.insert(name, value);
2644 }
2645 }
2646 }
2647 if self.policy.redirects == RedirectPolicy::Follow {
2650 return Err(redirect_error(
2651 "realtime establishment does not follow redirects",
2652 None,
2653 ));
2654 }
2655 let mut ws = connect_websocket(request, Instant::now() + CONNECT_TIMEOUT)?;
2656 let stream = match ws.get_mut() {
2657 MaybeTlsStream::Plain(stream) => stream.established(),
2658 MaybeTlsStream::Rustls(stream) => stream.get_mut().established(),
2659 _ => {
2660 return Err(TransportError::new(
2661 "transport.failed",
2662 "unsupported websocket stream",
2663 ))
2664 }
2665 }
2666 .map_err(|e| io_err("realtime clone failed", e))?;
2667 stream
2668 .set_nonblocking(true)
2669 .map_err(|e| io_err("realtime nonblocking failed", e))?;
2670 let poller = Arc::new(Poller::new().map_err(|e| io_err("realtime poller failed", e))?);
2671 unsafe { poller.add(&stream, Event::readable(0)) }
2673 .map_err(|e| io_err("realtime register failed", e))?;
2674 let registration = SocketRegistration {
2675 poller: Arc::clone(&poller),
2676 stream,
2677 };
2678 let (outgoing, queued) = mpsc::sync_channel::<Outgoing>(1);
2679 self.outgoing = Some(Box::new(SocketSender {
2680 queue: outgoing,
2681 poller,
2682 }));
2683 self.reader_stop = Arc::new(AtomicBool::new(false));
2684 self.round = Arc::new(RoundChannel::default());
2685 let inbound = Arc::clone(&self.inbound);
2686 let stop = Arc::clone(&self.reader_stop);
2687 let round = Arc::clone(&self.round);
2688 let silence = 2 * self
2693 .policy
2694 .realtime_heartbeat
2695 .unwrap_or(Duration::from_secs(30));
2696 self.reader = Some(std::thread::spawn(move || {
2697 let mut events = Events::new();
2698 let mut pending: Option<mpsc::Sender<Result<(), TransportError>>> = None;
2699 let mut heard: Option<Instant> = None;
2700 let failure = 'io: loop {
2701 if stop.load(Ordering::SeqCst) {
2702 let _ = ws.close(None);
2703 let _ = ws.flush();
2704 break TransportError::new(
2705 "sync.transport_failed",
2706 "realtime disconnected mid-round (§8.7)",
2707 );
2708 }
2709 if pending.is_none() {
2710 if let Ok(outgoing) = queued.try_recv() {
2711 pending = Some(outgoing.completed);
2712 if let Err(error) = ws.write(outgoing.message) {
2714 if !is_would_block(&error) {
2715 break ws_err("realtime write failed", error);
2716 }
2717 }
2718 }
2719 }
2720 let mut drained = false;
2722 for _ in 0..64 {
2723 let received = ws.read();
2724 if let Ok(message) = &received {
2725 let armed = heard.is_some()
2726 || matches!(message, Message::Text(text)
2727 if serde_json::from_str::<serde_json::Value>(text)
2728 .is_ok_and(|event| event["event"] == "heartbeat"));
2729 if armed {
2730 heard = Some(Instant::now());
2731 }
2732 let kind = match message {
2733 Message::Text(_) => Some(RealtimeReceiptKind::Text),
2734 Message::Binary(bytes) => Some(match bytes.first() {
2735 Some(&crate::REALTIME_TAG_DELTA) => RealtimeReceiptKind::Delta,
2736 Some(&crate::REALTIME_TAG_ROUND) => RealtimeReceiptKind::Round,
2737 _ => RealtimeReceiptKind::OtherBinary,
2738 }),
2739 Message::Ping(_) => Some(RealtimeReceiptKind::Ping),
2740 Message::Pong(_) => Some(RealtimeReceiptKind::Pong),
2741 Message::Close(_) => Some(RealtimeReceiptKind::Close),
2742 Message::Frame(_) => None,
2743 };
2744 if let Some(kind) = kind {
2745 let bytes = match message {
2746 Message::Close(Some(close)) => 2 + close.reason.len(),
2747 _ => message.len(),
2748 };
2749 inbound.observe(kind, bytes);
2750 }
2751 }
2752 match received {
2753 Ok(Message::Text(text)) => {
2754 inbound.push(Inbound::Text(text.to_string()))
2755 }
2756 Ok(Message::Binary(bytes)) => {
2757 if let Some(delta) = round.route_binary(&bytes) {
2758 inbound.push(Inbound::Binary(delta));
2759 }
2760 }
2761 Err(error) if is_would_block(&error) => {
2762 drained = true;
2763 break;
2764 }
2765 Ok(Message::Close(_)) => {
2766 let _ = ws.flush();
2767 break 'io TransportError::new(
2768 "sync.transport_failed",
2769 "realtime disconnected mid-round (§8.7)",
2770 );
2771 }
2772 Err(_) => {
2773 break 'io TransportError::new(
2774 "sync.transport_failed",
2775 "realtime disconnected mid-round (§8.7)",
2776 );
2777 }
2778 Ok(_) => {}
2779 }
2780 if inbound.overflowed() {
2783 let _ = ws.close(None);
2784 let _ = ws.flush();
2785 let mut error = TransportError::new(
2786 "transport.failed",
2787 "realtime inbound buffer is full",
2788 );
2789 error.details = Some(serde_json::json!({
2790 "causeKind": "inbound_overflow",
2791 "limit": super::INBOUND_BUFFER_LIMIT_BYTES,
2792 }));
2793 break 'io error;
2794 }
2795 }
2796 let interest = match ws.flush() {
2797 Ok(()) => {
2798 if let Some(completed) = pending.take() {
2799 let _ = completed.send(Ok(()));
2800 }
2801 Event::readable(0)
2802 }
2803 Err(error) if is_would_block(&error) => Event::all(0),
2804 Err(error) => break ws_err("realtime flush failed", error),
2805 };
2806 if !drained {
2807 continue;
2808 }
2809 if let Err(error) = registration.poller.modify(®istration.stream, interest) {
2810 break io_err("realtime rearm failed", error);
2811 }
2812 let wait = heard
2813 .map(|heard| (heard + silence).saturating_duration_since(Instant::now()));
2814 if wait.is_some_and(|wait| wait.is_zero()) {
2815 break timeout_error("realtime heartbeat missed", "heartbeat");
2816 }
2817 events.clear();
2818 if let Err(error) = registration.poller.wait(&mut events, wait) {
2819 if error.kind() != std::io::ErrorKind::Interrupted {
2820 break io_err("realtime wait failed", error);
2821 }
2822 }
2823 };
2824 if !stop.load(Ordering::SeqCst) {
2829 inbound.push(Inbound::Disconnected(failure.clone()));
2830 }
2831 if let Some(completed) = pending {
2832 let _ = completed.send(Err(failure.clone()));
2833 }
2834 while let Ok(outgoing) = queued.try_recv() {
2835 let _ = outgoing.completed.send(Err(failure.clone()));
2836 }
2837 round.fail_in_flight(failure);
2838 }));
2839 Ok(())
2840 }
2841
2842 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
2843 self.send_message(Message::Text(text.to_owned().into()))
2844 }
2845
2846 fn realtime_close(&mut self) -> Result<(), TransportError> {
2847 self.shutdown();
2848 Ok(())
2849 }
2850
2851 fn round_deadline(&self) -> Option<Duration> {
2852 NativeTransport::round_deadline(self)
2853 }
2854
2855 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
2856 NativeTransport::set_round_deadline(self, deadline);
2857 }
2858 }
2859 #[cfg(test)]
2860 mod transfer_tests {
2861 use super::*;
2862 use ureq::unversioned::resolver::DefaultResolver;
2863 use ureq::unversioned::transport::{ConnectionDetails, Connector};
2864
2865 const SECRET: &str = "https://user:password@cdn.example/secret-capability-path?secret-query=value#secret-fragment";
2866
2867 #[derive(Debug)]
2868 struct SecretError;
2869 impl std::fmt::Display for SecretError {
2870 fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2871 panic!("transport error must never be formatted")
2872 }
2873 }
2874 impl std::error::Error for SecretError {}
2875
2876 #[derive(Debug)]
2877 struct FailingConnector(&'static str);
2878 impl Connector for FailingConnector {
2879 type Out = ();
2880 fn connect(
2881 &self,
2882 _: &ConnectionDetails,
2883 _: Option<()>,
2884 ) -> Result<Option<()>, ureq::Error> {
2885 Err(match self.0 {
2886 "io" => ureq::Error::Io(std::io::Error::new(
2887 std::io::ErrorKind::ConnectionRefused,
2888 format!("404 {SECRET}"),
2889 )),
2890 "tls" => ureq::Error::Tls(SECRET),
2891 "request" => ureq::Error::BadUri(SECRET.into()),
2892 "timeout" => ureq::Error::Timeout(ureq::Timeout::Global),
2893 "404" => ureq::Error::StatusCode(404),
2894 "401" => ureq::Error::StatusCode(401),
2895 "403" => ureq::Error::StatusCode(403),
2896 "503" => ureq::Error::StatusCode(503),
2897 _ => ureq::Error::Other(Box::new(SecretError)),
2898 })
2899 }
2900 }
2901
2902 #[test]
2903 fn injected_http_failures_never_expose_capabilities_or_format_the_cause() {
2904 for kind in [
2905 "io", "tls", "request", "timeout", "unknown", "404", "401", "403", "503",
2906 ] {
2907 for operation in [
2908 "sync",
2909 "remote",
2910 "segment",
2911 "signed_segment",
2912 "blob",
2913 "signed_blob",
2914 "upload",
2915 "grant",
2916 "signed_put",
2917 ] {
2918 let mut transport =
2919 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None)
2920 .unwrap();
2921 transport.agent = ureq::Agent::with_parts(
2922 ureq::Agent::config_builder().build(),
2923 FailingConnector(kind),
2924 DefaultResolver::default(),
2925 );
2926 let mut progress = |_| {};
2927 let result = match operation {
2928 "sync" => transport.sync(&[]).map(|_| ()),
2929 "remote" => transport.remote_operation(&[]).map(|_| ()),
2930 "segment" => transport.download_segment(&SegmentRequest { segment_id: "sha256:test".into(), table: "tasks".into(), requested_scopes_json: "{}".into(), byte_length: 1 << 20 }, &mut progress).map(|_| ()),
2931 "signed_segment" => transport.fetch_url("http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment", 1 << 20, &mut progress).map(|_| ()),
2932 "blob" => transport.blob_download("sha256:test", None).map(|_| ()),
2933 "signed_blob" => transport.fetch_blob_url("http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment", None).map(|_| ()),
2934 "upload" => transport.blob_upload("sha256:test", &[], None),
2935 "grant" => transport.blob_upload_grant("sha256:test", 0, None).map(|_| ()),
2936 _ => transport.blob_put_url(&format!("sha256:{}", "0".repeat(64)), "http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment", &[], None),
2937 };
2938 let error = result.unwrap_err();
2939 let expected_code = match (operation, kind) {
2940 (_, "timeout") => "transport.timeout",
2941 ("segment" | "signed_segment", _) => "sync.transport_failed",
2942 ("blob", "404") => "blob.not_found",
2943 ("blob", "401") => "sync.auth_required",
2944 ("blob", "403") => "blob.forbidden",
2945 _ => "transport.failed",
2946 };
2947 assert_eq!(error.code, expected_code, "{operation} {kind}");
2948 let details = error.details.unwrap();
2949 assert_eq!(
2950 details["causeKind"],
2951 match kind {
2952 "io" => "connect",
2953 "404" | "401" | "403" | "503" => "status",
2954 other => other,
2955 }
2956 );
2957 assert_eq!(
2958 details.get("httpStatus").and_then(|value| value.as_u64()),
2959 kind.parse::<u64>().ok()
2960 );
2961 let rendered = format!("{}{}", error.message, details);
2962 for secret in [
2963 "http://",
2964 "https://",
2965 "password",
2966 "secret-capability-path",
2967 "secret-query",
2968 "secret-fragment",
2969 ] {
2970 assert!(!rendered.contains(secret), "{operation} {kind}");
2971 }
2972 assert!(details.get("causeMessage").is_none());
2973 assert!(details.get("path").is_none());
2974 }
2975 }
2976 }
2977
2978 #[test]
2979 fn injected_body_and_websocket_errors_do_not_expose_payloads_or_source_chains() {
2980 struct FailingRead;
2981 impl std::io::Read for FailingRead {
2982 fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
2983 Err(std::io::Error::other(SECRET))
2984 }
2985 }
2986 let response = ureq::http::Response::builder()
2987 .status(200)
2988 .body(ureq::Body::builder().reader(FailingRead))
2989 .unwrap();
2990 let error = segment_error(read_body(response, None, None, None).unwrap_err());
2991 assert_eq!(error.code, "sync.transport_failed");
2992 assert_eq!(error.message, "segment transfer failed");
2993 assert_eq!(
2994 error.details,
2995 Some(serde_json::json!({ "causeKind": "body", "httpStatus": 200 }))
2996 );
2997 for cause in [
2998 tungstenite::Error::Io(std::io::Error::other(SecretError)),
2999 tungstenite::Error::Http(Box::new(
3000 tungstenite::http::Response::builder()
3001 .status(403)
3002 .header("location", SECRET)
3003 .body(Some(SECRET.as_bytes().to_vec()))
3004 .unwrap(),
3005 )),
3006 ] {
3007 let error = ws_err("realtime connection failed", cause);
3008 assert_eq!(error.message, "realtime connection failed");
3009 let details = error.details.unwrap();
3010 assert!(details.get("path").is_none());
3011 assert!(details.get("causeMessage").is_none());
3012 assert!(!details.to_string().contains("secret"));
3013 }
3014 }
3015
3016 #[test]
3017 fn body_read_timeouts_are_classified_through_ureq_wrapping() {
3018 let direct = ureq::Error::Timeout(ureq::Timeout::Global).into_io();
3020 assert!(is_timeout_io(&direct));
3021 let wrapped = ureq::Error::Decompress(
3023 "gzip",
3024 ureq::Error::Timeout(ureq::Timeout::Global).into_io(),
3025 )
3026 .into_io();
3027 assert!(is_timeout_io(&wrapped));
3028 assert!(is_timeout_io(&std::io::Error::new(
3030 std::io::ErrorKind::TimedOut,
3031 "plain"
3032 )));
3033 assert!(!is_timeout_io(&std::io::Error::other("other")));
3034 assert!(!is_timeout_io(&ureq::Error::StatusCode(500).into_io()));
3035 }
3036
3037 #[test]
3038 fn body_read_wrapped_timeouts_surface_the_typed_code() {
3039 struct TimeoutRead(bool);
3040 impl std::io::Read for TimeoutRead {
3041 fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
3042 Err(if self.0 {
3043 ureq::Error::Timeout(ureq::Timeout::Global).into_io()
3044 } else {
3045 ureq::Error::Decompress(
3046 "gzip",
3047 ureq::Error::Timeout(ureq::Timeout::Global).into_io(),
3048 )
3049 .into_io()
3050 })
3051 }
3052 }
3053 for compressed in [false, true] {
3054 let response = ureq::http::Response::builder()
3055 .status(200)
3056 .body(ureq::Body::builder().reader(TimeoutRead(compressed)))
3057 .unwrap();
3058 let error = read_body(response, None, None, None).unwrap_err();
3059 assert_eq!(error.code, "transport.timeout", "{compressed}");
3060 assert_eq!(
3061 error.details,
3062 Some(serde_json::json!({ "causeKind": "timeout" }))
3063 );
3064 }
3065 }
3066
3067 #[test]
3068 fn policy_validation_rejects_zero_and_unrepresentable_bounds() {
3069 assert_eq!(
3070 HostTransportPolicy {
3071 request_timeout: Some(Duration::ZERO),
3072 ..Default::default()
3073 }
3074 .validate()
3075 .unwrap_err(),
3076 "sync.invalid_request: transport config requestTimeoutMs is out of range"
3077 );
3078 assert_eq!(
3079 HostTransportPolicy {
3080 round_deadline: Some(Duration::MAX),
3081 ..Default::default()
3082 }
3083 .validate()
3084 .unwrap_err(),
3085 "sync.invalid_request: transport config roundDeadlineMs is out of range"
3086 );
3087 for interval in [Duration::ZERO, Duration::MAX / 2 + Duration::from_secs(1)] {
3088 assert_eq!(
3089 HostTransportPolicy {
3090 realtime_heartbeat: Some(interval),
3091 ..Default::default()
3092 }
3093 .validate()
3094 .unwrap_err(),
3095 "sync.invalid_request: transport config realtimeHeartbeatMs is out of range"
3096 );
3097 }
3098 assert!(HostTransportPolicy {
3099 max_request_bytes: Some(0),
3100 ..Default::default()
3101 }
3102 .validate()
3103 .is_err());
3104 assert!(HostTransportPolicy {
3105 max_response_bytes: Some(0),
3106 ..Default::default()
3107 }
3108 .validate()
3109 .is_err());
3110 assert!(HostTransportPolicy::default().validate().is_ok());
3113 assert_eq!(
3114 HostTransportPolicy::from_config(&serde_json::json!({ "requestTimeoutMs": 0 }))
3115 .unwrap_err(),
3116 "sync.invalid_request: transport config requestTimeoutMs must be a positive integer"
3117 );
3118 }
3119
3120 #[test]
3121 fn set_policy_rejects_invalid_and_preserves_the_current_policy() {
3122 let mut transport =
3123 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None).unwrap();
3124 assert_eq!(transport.policy.max_response_bytes, None);
3125 let error = transport
3126 .set_policy(HostTransportPolicy {
3127 max_response_bytes: Some(0),
3128 ..Default::default()
3129 })
3130 .unwrap_err();
3131 assert_eq!(
3132 error,
3133 "sync.invalid_request: transport config maxResponseBytes must be a positive integer"
3134 );
3135 assert_eq!(transport.policy.max_response_bytes, None);
3136 }
3137
3138 #[test]
3139 fn an_expired_round_deadline_is_typed_and_cleared_for_the_next_round() {
3140 let mut transport =
3141 NativeTransport::new("http://127.0.0.1:9", &serde_json::json!({}), None).unwrap();
3142 transport.set_round_deadline(Some(std::time::Instant::now() - Duration::from_secs(1)));
3143 let error = transport.blob_download("sha256:test", None).unwrap_err();
3144 assert_eq!(error.code, "transport.timeout");
3145 assert_eq!(
3146 error.details,
3147 Some(serde_json::json!({ "causeKind": "round_deadline" }))
3148 );
3149 transport.set_round_deadline(None);
3151 assert!(transport.round_remaining().is_ok());
3152 }
3153
3154 #[test]
3155 fn an_expired_round_deadline_refuses_before_queuing_a_realtime_send() {
3156 let mut transport =
3157 NativeTransport::new("http://127.0.0.1:9", &serde_json::json!({}), None).unwrap();
3158 transport.set_round_deadline(Some(std::time::Instant::now() - Duration::from_secs(1)));
3159 let error = transport.realtime_send("blocked").unwrap_err();
3163 assert_eq!(error.code, "transport.timeout");
3164 assert_eq!(
3165 error.details,
3166 Some(serde_json::json!({ "causeKind": "round_deadline" }))
3167 );
3168 transport.set_round_deadline(None);
3171 let error = transport.realtime_send("blocked").unwrap_err();
3172 assert_eq!(error.code, "transport.failed");
3173 }
3174
3175 #[test]
3176 fn the_request_timeout_bounds_phases_and_the_round_deadline_bounds_the_call() {
3177 #[derive(Debug)]
3178 struct TimeoutRecording(Arc<Mutex<Vec<Option<Duration>>>>);
3179 impl Connector for TimeoutRecording {
3180 type Out = ();
3181 fn connect(
3182 &self,
3183 details: &ConnectionDetails,
3184 _: Option<()>,
3185 ) -> Result<Option<()>, ureq::Error> {
3186 self.0
3187 .lock()
3188 .unwrap()
3189 .push(details.config.timeouts().global);
3190 Err(ureq::Error::ConnectionFailed)
3191 }
3192 }
3193 let mut transport =
3194 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None).unwrap();
3195 transport
3196 .set_policy(HostTransportPolicy {
3197 request_timeout: Some(Duration::from_millis(1234)),
3198 ..Default::default()
3199 })
3200 .unwrap();
3201 let timeouts = transport.agent.config().timeouts();
3202 for phase in [
3203 timeouts.resolve,
3204 timeouts.connect,
3205 timeouts.send_request,
3206 timeouts.recv_response,
3207 ] {
3208 assert_eq!(phase, Some(Duration::from_millis(1234)));
3209 }
3210 assert_eq!(timeouts.global, None, "the request bound is not end to end");
3211 assert_eq!(timeouts.recv_body, None, "a body is bounded by silence");
3212 let seen = Arc::new(Mutex::new(Vec::new()));
3213 transport.agent = ureq::Agent::with_parts(
3214 ureq::Agent::config_builder().build(),
3215 TimeoutRecording(Arc::clone(&seen)),
3216 DefaultResolver::default(),
3217 );
3218 let _ = transport.blob_upload("sha256:test", &[0u8; 4], None);
3219 transport.set_round_deadline(Some(std::time::Instant::now() + Duration::from_secs(60)));
3220 let _ = transport.blob_upload("sha256:test", &[0u8; 4], None);
3221 let seen = seen.lock().unwrap();
3222 assert_eq!(seen[0], None, "no round deadline, no call ceiling");
3223 assert!(seen[1].is_some_and(|ceiling| ceiling <= Duration::from_secs(60)));
3224 }
3225
3226 #[test]
3227 fn the_idle_bound_replaces_an_unbounded_or_longer_wait_and_keeps_a_shorter_one() {
3228 let idle = Duration::from_secs(30);
3229 let unbounded = NextTimeout::default();
3230 assert_eq!(idle_bound(unbounded, idle).after, UreqDuration::Exact(idle));
3231 let longer = NextTimeout {
3232 after: UreqDuration::from_secs(90),
3233 per_read: Some(UreqDuration::from_secs(7)),
3234 ..NextTimeout::default()
3235 };
3236 let bounded = idle_bound(longer, idle);
3237 assert_eq!(bounded.after, UreqDuration::Exact(idle));
3238 assert_eq!(bounded.reason, UreqTimeout::RecvBody);
3239 assert_eq!(
3240 bounded.per_read, longer.per_read,
3241 "per-read cap passes through"
3242 );
3243 let shorter = NextTimeout {
3244 after: UreqDuration::from_secs(5),
3245 ..NextTimeout::default()
3246 };
3247 assert_eq!(idle_bound(shorter, idle), shorter);
3248 }
3249
3250 fn shaped_link(
3255 total: u64,
3256 chunk: u64,
3257 latency: Duration,
3258 rate: u64,
3259 stalls: &[(Duration, Duration)],
3260 ) -> Vec<Duration> {
3261 let mut at = latency;
3262 let mut arrivals = Vec::new();
3263 let mut left = total;
3264 while left > 0 {
3265 let piece = left.min(chunk);
3266 at += Duration::from_micros(piece * 1_000_000 / rate);
3267 for (start, length) in stalls {
3268 if at > *start && at < *start + *length {
3269 at = *start + *length;
3270 }
3271 }
3272 arrivals.push(at);
3273 left -= piece;
3274 }
3275 arrivals
3276 }
3277
3278 fn round_over(
3282 arrivals: &[Duration],
3283 idle: Duration,
3284 ceiling: Option<Duration>,
3285 ) -> Result<Duration, (Duration, bool)> {
3286 let base = Instant::now();
3287 let mut last = base;
3288 for arrival in arrivals {
3289 let (deadline, ceiling_binds) =
3290 round_expiry(last, idle, ceiling.map(|ceiling| base + ceiling));
3291 if base + *arrival > deadline {
3292 return Err((deadline - base, ceiling_binds));
3293 }
3294 last = base + *arrival;
3295 }
3296 Ok(last - base)
3297 }
3298
3299 #[test]
3300 fn a_slow_link_that_keeps_delivering_never_trips_the_idle_bound() {
3301 let link = shaped_link(
3303 8_600_000,
3304 64 * 1024,
3305 Duration::from_millis(80),
3306 100_000,
3307 &[],
3308 );
3309 let finished = round_over(&link, ROUND_IDLE_TIMEOUT, None).unwrap();
3310 assert!(
3311 finished > ROUND_IDLE_TIMEOUT * 2,
3312 "the transfer outlasts the old whole-transfer bound ({finished:?})"
3313 );
3314 }
3315
3316 #[test]
3317 fn a_stall_shorter_than_the_idle_bound_is_survived_and_a_longer_one_is_not() {
3318 let link = |length| {
3319 shaped_link(
3320 4_000_000,
3321 64 * 1024,
3322 Duration::from_millis(80),
3323 100_000,
3324 &[(Duration::from_secs(10), length)],
3325 )
3326 };
3327 round_over(&link(Duration::from_secs(29)), ROUND_IDLE_TIMEOUT, None).unwrap();
3328 let (expired, ceiling_bound) =
3329 round_over(&link(Duration::from_secs(31)), ROUND_IDLE_TIMEOUT, None).unwrap_err();
3330 assert!(!ceiling_bound);
3331 assert!(
3332 expired >= Duration::from_secs(30) && expired < Duration::from_secs(41),
3333 "the bound counts from the last chunk before the stall ({expired:?})"
3334 );
3335 }
3336
3337 #[test]
3338 fn the_host_ceiling_still_bounds_a_link_that_keeps_delivering() {
3339 let link = shaped_link(
3340 8_600_000,
3341 64 * 1024,
3342 Duration::from_millis(80),
3343 100_000,
3344 &[],
3345 );
3346 let (expired, ceiling_bound) =
3347 round_over(&link, ROUND_IDLE_TIMEOUT, Some(Duration::from_secs(60))).unwrap_err();
3348 assert!(ceiling_bound);
3349 assert_eq!(expired, Duration::from_secs(60));
3350 }
3351
3352 #[test]
3353 fn an_expired_round_deadline_bounds_the_socket_wait() {
3354 let channel = RoundChannel::default();
3355 let error = channel
3356 .wait(
3357 Some(std::time::Instant::now() - Duration::from_secs(1)),
3358 ROUND_IDLE_TIMEOUT,
3359 )
3360 .unwrap_err();
3361 assert_eq!(error.code, "transport.timeout");
3362 assert_eq!(
3363 error.details,
3364 Some(serde_json::json!({ "causeKind": "round_deadline" }))
3365 );
3366 }
3367 }
3368}
3369
3370#[cfg(all(test, feature = "native-transport"))]
3371mod tests {
3372 use super::*;
3373 use std::io::{BufRead, BufReader, Write};
3374
3375 #[test]
3376 fn an_undrained_inbound_buffer_past_its_limit_ends_the_connection() {
3377 const DELTA_BYTES: usize = 1024 * 1024;
3378 let deltas = INBOUND_BUFFER_LIMIT_BYTES / DELTA_BYTES + 1;
3379 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3380 let base = format!("http://{}", listener.local_addr().unwrap());
3381 let server = std::thread::spawn(move || {
3382 let (socket, _) = listener.accept().unwrap();
3383 let mut socket = tungstenite::accept(socket).unwrap();
3384 for _ in 0..deltas {
3385 let mut frame = vec![crate::REALTIME_TAG_DELTA];
3386 frame.resize(DELTA_BYTES + 1, 7);
3387 socket
3388 .send(tungstenite::Message::Binary(frame.into()))
3389 .unwrap();
3390 }
3391 while !matches!(socket.read(), Ok(tungstenite::Message::Close(_)) | Err(_)) {}
3392 });
3393 let (woke, wakes) = mpsc::channel::<()>();
3394 let mut transport = HostTransport::from_config_with_notify(
3395 &serde_json::json!({"baseUrl": base}),
3396 Some(Arc::new(move || {
3397 let _ = woke.send(());
3398 })),
3399 )
3400 .unwrap();
3401 transport.realtime_connect("c1").unwrap();
3402 for _ in 0..=deltas {
3404 wakes.recv().unwrap();
3405 }
3406 let inbound = transport.take_inbound();
3407 assert_eq!(inbound.len(), deltas + 1);
3408 assert!(inbound[..deltas]
3409 .iter()
3410 .all(|frame| matches!(frame, Inbound::Binary(bytes) if bytes.len() == DELTA_BYTES)));
3411 assert!(matches!(
3412 &inbound[deltas],
3413 Inbound::Disconnected(error) if error.code == "transport.failed"
3414 && error.details.as_ref().and_then(|d| d.get("causeKind"))
3415 == Some(&serde_json::json!("inbound_overflow"))
3416 ));
3417 transport.shutdown();
3418 server.join().unwrap();
3419 }
3420
3421 #[test]
3422 fn the_first_heartbeat_arms_silence_detection() {
3423 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3424 let base = format!("http://{}", listener.local_addr().unwrap());
3425 let (beat, heartbeat) = mpsc::channel::<()>();
3426 let server = std::thread::spawn(move || {
3427 let (socket, _) = listener.accept().unwrap();
3428 let mut socket = tungstenite::accept(socket).unwrap();
3429 socket
3430 .send(tungstenite::Message::Text("{\"event\":\"hello\"}".into()))
3431 .unwrap();
3432 heartbeat.recv().unwrap();
3433 socket
3434 .send(tungstenite::Message::Text(
3435 "{\"event\":\"heartbeat\",\"data\":{\"timestamp\":1}}".into(),
3436 ))
3437 .unwrap();
3438 while socket.read().is_ok() {}
3440 });
3441 let (woke, wakes) = mpsc::channel::<()>();
3442 let mut transport = HostTransport::from_config_with_notify(
3443 &serde_json::json!({"baseUrl": base, "realtimeHeartbeatMs": 50}),
3444 Some(Arc::new(move || {
3445 let _ = woke.send(());
3446 })),
3447 )
3448 .unwrap();
3449 transport.realtime_connect("c1").unwrap();
3450 wakes.recv_timeout(Duration::from_secs(10)).unwrap();
3451 assert!(wakes.recv_timeout(Duration::from_millis(250)).is_err());
3454 beat.send(()).unwrap();
3455 for _ in 0..2 {
3456 wakes.recv_timeout(Duration::from_secs(10)).unwrap();
3457 }
3458 let inbound = transport.take_inbound();
3459 assert_eq!(inbound.len(), 3);
3460 assert!(matches!(&inbound[0], Inbound::Text(_)));
3461 assert!(matches!(&inbound[1], Inbound::Text(text) if text.contains("heartbeat")));
3462 assert!(matches!(
3463 &inbound[2],
3464 Inbound::Disconnected(error) if error.code == "transport.timeout"
3465 && error.details == Some(serde_json::json!({ "causeKind": "heartbeat" }))
3466 ));
3467 transport.shutdown();
3468 server.join().unwrap();
3469 }
3470
3471 #[test]
3472 fn segment_and_blob_reads_stop_past_the_declared_byte_length() {
3473 for case in ["segment", "signed_segment", "blob", "signed_blob"] {
3474 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3475 let base = format!("http://{}", listener.local_addr().unwrap());
3476 let server = std::thread::spawn(move || {
3477 let (mut socket, _) = listener.accept().unwrap();
3478 let mut reader = BufReader::new(socket.try_clone().unwrap());
3479 loop {
3480 let mut line = String::new();
3481 reader.read_line(&mut line).unwrap();
3482 if line == "\r\n" {
3483 break;
3484 }
3485 }
3486 let body = vec![7u8; 4 << 20];
3487 let _ = write!(
3489 socket,
3490 "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
3491 body.len()
3492 );
3493 let _ = socket.write_all(&body);
3494 });
3495 let mut transport =
3496 HostTransport::from_config(&serde_json::json!({ "baseUrl": base })).unwrap();
3497 let mut progress = |_| {};
3498 let bytes = match case {
3499 "segment" => transport.download_segment(
3500 &SegmentRequest {
3501 segment_id: "sha256:test".into(),
3502 table: "tasks".into(),
3503 requested_scopes_json: "{}".into(),
3504 byte_length: 16,
3505 },
3506 &mut progress,
3507 ),
3508 "signed_segment" => {
3509 transport.fetch_url(&format!("{base}/signed"), 16, &mut progress)
3510 }
3511 "blob" => transport
3512 .blob_download("sha256:test", Some(16))
3513 .map(|download| match download {
3514 BlobDownload::Bytes(bytes) => bytes,
3515 BlobDownload::Url { .. } => panic!("expected inline bytes"),
3516 }),
3517 _ => transport.fetch_blob_url(&format!("{base}/signed"), Some(16)),
3518 }
3519 .unwrap();
3520 assert!(
3523 bytes.len() > 16 && bytes.len() <= 16 + 64 * 1024,
3524 "{case} read {} bytes",
3525 bytes.len()
3526 );
3527 server.join().unwrap();
3528 }
3529 }
3530
3531 #[test]
3532 fn a_round_deadline_expiring_during_a_socket_send_is_the_typed_timeout() {
3533 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3534 let base = format!("http://{}", listener.local_addr().unwrap());
3535 let (release, released) = mpsc::channel::<()>();
3536 let server = std::thread::spawn(move || {
3539 let (socket, _) = listener.accept().unwrap();
3540 let _socket = tungstenite::accept(socket).unwrap();
3541 let _ = released.recv();
3542 });
3543 let mut transport =
3544 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3545 transport.realtime_connect("c1").unwrap();
3546 transport.set_round_deadline(Some(std::time::Instant::now() + Duration::from_millis(200)));
3547 let error = transport
3548 .realtime_sync(&vec![0u8; 32 * 1024 * 1024])
3549 .unwrap_err();
3550 assert_eq!(error.code, "transport.timeout");
3551 assert_eq!(
3552 error.details,
3553 Some(serde_json::json!({ "causeKind": "round_deadline" }))
3554 );
3555 assert!(matches!(
3556 &transport.take_inbound()[..],
3557 [Inbound::Disconnected(lost)] if lost.code == "transport.timeout"
3558 ));
3559 transport.shutdown();
3560 release.send(()).unwrap();
3561 server.join().unwrap();
3562 }
3563
3564 #[test]
3565 fn receipt_observer_covers_round_chunks_wakes_reconnect_and_overflow() {
3566 use ssp2::model::{Frame, Message, MsgKind};
3567 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3568 let base = format!("http://{}", listener.local_addr().unwrap());
3569 let response = ssp2::encode_message(&Message {
3570 wire_version: ssp2::decode::WIRE_VERSION,
3571 msg_kind: MsgKind::Response,
3572 frames: vec![Frame::RespHeader {
3573 required_schema_version: None,
3574 latest_schema_version: None,
3575 log_epoch: "epoch".into(),
3576 reset_required: false,
3577 }],
3578 });
3579 let server_response = response.clone();
3580 let server = std::thread::spawn(move || {
3581 for _ in 0..2 {
3582 let (socket, _) = listener.accept().unwrap();
3583 let mut socket = tungstenite::accept(socket).unwrap();
3584 assert!(matches!(
3585 socket.read().unwrap(),
3586 tungstenite::Message::Binary(_)
3587 ));
3588 socket
3589 .send(tungstenite::Message::Binary(
3590 vec![crate::REALTIME_TAG_DELTA, 7].into(),
3591 ))
3592 .unwrap();
3593 socket
3594 .send(tungstenite::Message::Text("wake".into()))
3595 .unwrap();
3596 let split = server_response.len() / 2;
3597 for part in [&server_response[..split], &server_response[split..]] {
3598 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
3599 chunk.extend_from_slice(part);
3600 socket
3601 .send(tungstenite::Message::Binary(chunk.into()))
3602 .unwrap();
3603 }
3604 assert!(matches!(
3605 socket.read().unwrap(),
3606 tungstenite::Message::Close(_)
3607 ));
3608 }
3609 });
3610 let mut transport =
3611 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3612 let (sender, receiver) = mpsc::sync_channel(4);
3613 let (small_sender, small_receiver) = mpsc::sync_channel(1);
3614 transport.set_realtime_receipt_observer(Some(sender));
3615 for round in 0..2 {
3616 transport.realtime_connect("c1").unwrap();
3617 let before = SystemTime::now();
3618 assert_eq!(transport.realtime_sync(b"request").unwrap(), response);
3619 let after = SystemTime::now();
3620 if round == 0 {
3621 let receipts = [
3622 receiver.recv().unwrap(),
3623 receiver.recv().unwrap(),
3624 receiver.recv().unwrap(),
3625 receiver.recv().unwrap(),
3626 ];
3627 assert_eq!(
3628 receipts.map(|r| r.kind),
3629 [
3630 RealtimeReceiptKind::Delta,
3631 RealtimeReceiptKind::Text,
3632 RealtimeReceiptKind::Round,
3633 RealtimeReceiptKind::Round
3634 ]
3635 );
3636 assert_eq!(
3637 receipts.map(|r| r.bytes),
3638 [
3639 2,
3640 4,
3641 1 + response.len() / 2,
3642 1 + response.len() - response.len() / 2
3643 ]
3644 );
3645 assert!(receipts
3646 .iter()
3647 .all(|r| r.received_at >= before && r.received_at <= after));
3648 assert!(
3649 matches!(&transport.take_inbound()[..], [Inbound::Binary(bytes), Inbound::Text(text)] if bytes == &[7] && text == "wake")
3650 );
3651 assert_eq!(transport.take_dropped_realtime_receipts(), 0);
3652 transport.set_realtime_receipt_observer(Some(small_sender.clone()));
3655 } else {
3656 assert_eq!(
3657 small_receiver.recv().unwrap().kind,
3658 RealtimeReceiptKind::Delta
3659 );
3660 assert_eq!(transport.take_dropped_realtime_receipts(), 3);
3661 assert_eq!(transport.take_dropped_realtime_receipts(), 0);
3662 }
3663 transport.shutdown();
3664 }
3665 server.join().unwrap();
3666 }
3667
3668 #[test]
3669 fn receipt_observer_reports_chunks_of_a_failed_round() {
3670 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3671 let base = format!("http://{}", listener.local_addr().unwrap());
3672 let server = std::thread::spawn(move || {
3673 let (socket, _) = listener.accept().unwrap();
3674 let mut socket = tungstenite::accept(socket).unwrap();
3675 assert!(matches!(
3676 socket.read().unwrap(),
3677 tungstenite::Message::Binary(_)
3678 ));
3679 socket
3680 .send(tungstenite::Message::Binary(
3681 vec![crate::REALTIME_TAG_ROUND, b'S'].into(),
3682 ))
3683 .unwrap();
3684 socket
3685 .close(Some(tungstenite::protocol::CloseFrame {
3686 code: tungstenite::protocol::frame::coding::CloseCode::Normal,
3687 reason: "bye".into(),
3688 }))
3689 .unwrap();
3690 });
3691 let mut transport =
3692 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3693 let (sender, receiver) = mpsc::sync_channel(2);
3694 transport.set_realtime_receipt_observer(Some(sender));
3695 transport.realtime_connect("c1").unwrap();
3696 assert!(transport.realtime_sync(b"request").is_err());
3697 assert_eq!(receiver.recv().unwrap().kind, RealtimeReceiptKind::Round);
3698 let close = receiver.recv().unwrap();
3699 assert_eq!(close.kind, RealtimeReceiptKind::Close);
3700 assert_eq!(close.bytes, 5);
3701 assert_eq!(transport.take_dropped_realtime_receipts(), 0);
3702 transport.shutdown();
3703 server.join().unwrap();
3704 }
3705
3706 #[test]
3707 fn detached_round_uses_the_owned_socket_and_shutdown_releases_it() {
3708 use ssp2::model::{Frame, Message, MsgKind};
3709 use std::sync::mpsc::channel;
3710 for cancelled in [false, true] {
3711 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3712 let base = format!("http://{}", listener.local_addr().unwrap());
3713 let (entered_tx, entered_rx) = channel();
3714 let (release_tx, release_rx) = channel();
3715 let server = std::thread::spawn(move || {
3716 let (socket, _) = listener.accept().unwrap();
3717 let mut socket = tungstenite::accept(socket).unwrap();
3718 let request = socket.read().unwrap().into_data();
3719 assert_eq!(request[0], crate::REALTIME_TAG_ROUND);
3720 assert_eq!(
3721 ssp2::decode_message(&request[1..]).unwrap().msg_kind,
3722 MsgKind::Request
3723 );
3724 entered_tx.send(()).unwrap();
3725 release_rx.recv().unwrap();
3726 if !cancelled {
3727 let response = ssp2::encode_message(&Message {
3728 wire_version: ssp2::decode::WIRE_VERSION,
3729 msg_kind: MsgKind::Response,
3730 frames: vec![Frame::RespHeader {
3731 required_schema_version: None,
3732 latest_schema_version: None,
3733 log_epoch: "epoch".into(),
3734 reset_required: true,
3735 }],
3736 });
3737 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
3738 chunk.extend(response);
3739 socket
3740 .send(tungstenite::Message::Binary(chunk.into()))
3741 .unwrap();
3742 }
3743 });
3744 let mut owner =
3745 HostTransport::from_config(&serde_json::json!({"baseUrl":base})).unwrap();
3746 owner.realtime_connect("detached-round").unwrap();
3747 let mut detached = owner.fork_round();
3748 let network = std::thread::spawn(move || {
3749 let request = ssp2::encode_message(&Message {
3750 wire_version: ssp2::decode::WIRE_VERSION,
3751 msg_kind: MsgKind::Request,
3752 frames: vec![
3753 Frame::ReqHeader {
3754 client_id: "detached-round".into(),
3755 schema_version: 1,
3756 log_epoch: None,
3757 outbox_id: None,
3758 },
3759 Frame::PullHeader {
3760 limit_commits: 0,
3761 limit_snapshot_rows: 0,
3762 max_snapshot_pages: 0,
3763 accept: 3,
3764 cursor: -1,
3765 registration_digest: ssp2::registration::registration_digest([]),
3766 echo_digest: None,
3767 },
3768 ],
3769 });
3770 detached.realtime_sync(&request)
3771 });
3772 entered_rx.recv().unwrap();
3773 owner.set_headers(vec![("authorization".into(), "Bearer rotated".into())]);
3774 assert!(owner.take_inbound().is_empty());
3775 if cancelled {
3776 owner.shutdown();
3777 }
3778 release_tx.send(()).unwrap();
3779 let response = network.join().unwrap();
3780 if cancelled {
3781 assert!(response.is_err());
3782 } else {
3783 assert_eq!(
3784 ssp2::decode_message(&response.unwrap()).unwrap().msg_kind,
3785 MsgKind::Response
3786 );
3787 owner.shutdown();
3788 }
3789 server.join().unwrap();
3790 }
3791 }
3792
3793 fn minimal_response() -> Vec<u8> {
3794 use ssp2::model::{Frame, Message, MsgKind};
3795 ssp2::encode_message(&Message {
3796 wire_version: ssp2::decode::WIRE_VERSION,
3797 msg_kind: MsgKind::Response,
3798 frames: vec![Frame::RespHeader {
3799 required_schema_version: None,
3800 latest_schema_version: None,
3801 log_epoch: "epoch".into(),
3802 reset_required: true,
3803 }],
3804 })
3805 }
3806
3807 #[test]
3808 fn a_presigned_blob_put_sends_the_blob_id_digest_without_hashing_the_body() {
3809 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3810 let base = format!("http://{}", listener.local_addr().unwrap());
3811 let server = std::thread::spawn(move || {
3812 let (mut socket, _) = listener.accept().unwrap();
3813 let mut reader = BufReader::new(socket.try_clone().unwrap());
3814 let mut head = Vec::new();
3815 loop {
3816 let mut line = String::new();
3817 reader.read_line(&mut line).unwrap();
3818 if line == "\r\n" {
3819 break;
3820 }
3821 let (name, value) = line.trim_end().split_once(": ").unwrap_or((&line, ""));
3822 head.push(format!("{}: {value}", name.to_ascii_lowercase()));
3823 }
3824 socket
3825 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
3826 .unwrap();
3827 head
3828 });
3829 let mut transport =
3830 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3831 let url = format!("{base}/bucket/key?X-Amz-Signature=x");
3832 let error = transport
3834 .blob_put_url("sha256:AB", &url, b"body", None)
3835 .unwrap_err();
3836 assert_eq!(error.code, "sync.invalid_request");
3837 transport
3840 .blob_put_url(&format!("sha256:{}", "ab".repeat(32)), &url, b"body", None)
3841 .unwrap();
3842 let head = server.join().unwrap();
3843 assert!(head.contains(
3844 &"x-amz-checksum-sha256: q6urq6urq6urq6urq6urq6urq6urq6urq6urq6urq6s=".to_owned()
3845 ));
3846 assert!(head.contains(&"content-length: 4".to_owned()));
3847 }
3848
3849 #[test]
3850 fn a_stalled_response_body_times_out_on_silence() {
3851 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3852 let base = format!("http://{}", listener.local_addr().unwrap());
3853 let server = std::thread::spawn(move || {
3854 let (mut socket, _) = listener.accept().unwrap();
3855 let mut reader = BufReader::new(socket.try_clone().unwrap());
3856 loop {
3857 let mut line = String::new();
3858 reader.read_line(&mut line).unwrap();
3859 if line == "\r\n" {
3860 break;
3861 }
3862 }
3863 socket
3864 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n0123456789")
3865 .unwrap();
3866 let mut rest = Vec::new();
3868 let _ = std::io::Read::read_to_end(&mut reader, &mut rest);
3869 });
3870 let mut transport = HostTransport::from_config(
3871 &serde_json::json!({"baseUrl": base, "requestTimeoutMs": 200}),
3872 )
3873 .unwrap();
3874 let mut received = 0;
3875 let error = transport
3876 .download_segment(
3877 &SegmentRequest {
3878 segment_id: "sha256:test".into(),
3879 table: "tasks".into(),
3880 requested_scopes_json: "{}".into(),
3881 byte_length: 1 << 20,
3882 },
3883 &mut |bytes| received = bytes,
3884 )
3885 .unwrap_err();
3886 assert_eq!(error.code, "transport.timeout");
3887 assert_eq!(
3888 error.details,
3889 Some(serde_json::json!({ "causeKind": "timeout" }))
3890 );
3891 server.join().unwrap();
3892 }
3893
3894 #[test]
3895 fn a_timed_out_socket_round_closes_the_socket_and_reports_the_loss() {
3896 use std::sync::mpsc::channel;
3897 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3898 let base = format!("http://{}", listener.local_addr().unwrap());
3899 let (closed_tx, closed_rx) = channel();
3900 let server = std::thread::spawn(move || {
3901 let (socket, _) = listener.accept().unwrap();
3902 let mut socket = tungstenite::accept(socket).unwrap();
3903 socket.read().unwrap();
3904 let response = minimal_response();
3906 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
3907 chunk.extend_from_slice(&response[..response.len() / 2]);
3908 socket
3909 .send(tungstenite::Message::Binary(chunk.into()))
3910 .unwrap();
3911 while let Ok(message) = socket.read() {
3913 if matches!(message, tungstenite::Message::Close(_)) {
3914 break;
3915 }
3916 }
3917 closed_tx.send(()).unwrap();
3918 });
3919 let mut transport =
3920 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3921 transport.realtime_connect("slow").unwrap();
3922 transport.set_round_deadline(Some(
3923 std::time::Instant::now() + std::time::Duration::from_millis(300),
3924 ));
3925 let error = transport.realtime_sync(b"request").unwrap_err();
3926 assert_eq!(error.code, "transport.timeout");
3927 closed_rx.recv().unwrap();
3928 let lost = transport
3929 .take_inbound()
3930 .into_iter()
3931 .filter(|inbound| matches!(inbound, Inbound::Disconnected(e) if e.code == "transport.timeout"))
3932 .count();
3933 assert_eq!(lost, 1, "the loss is reported once");
3934 server.join().unwrap();
3935 }
3936
3937 #[test]
3938 fn a_request_larger_than_a_chunk_rides_as_several_round_chunks() {
3939 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3940 let base = format!("http://{}", listener.local_addr().unwrap());
3941 let request: Vec<u8> = (0..3 * 64 * 1024 + 5).map(|i| (i % 251) as u8).collect();
3942 let expected = request.clone();
3943 let server = std::thread::spawn(move || {
3944 let (socket, _) = listener.accept().unwrap();
3945 let mut socket = tungstenite::accept(socket).unwrap();
3946 let mut body = Vec::new();
3947 let mut messages = 0;
3948 while body.len() < expected.len() {
3949 let message = socket.read().unwrap().into_data();
3950 assert_eq!(message[0], crate::REALTIME_TAG_ROUND);
3951 body.extend_from_slice(&message[1..]);
3952 messages += 1;
3953 }
3954 assert_eq!(body, expected);
3955 assert_eq!(messages, 4);
3956 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
3957 chunk.extend(minimal_response());
3958 socket
3959 .send(tungstenite::Message::Binary(chunk.into()))
3960 .unwrap();
3961 });
3962 let mut transport =
3963 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3964 transport.realtime_connect("chunked").unwrap();
3965 let response = transport.realtime_sync(&request).unwrap();
3966 assert_eq!(response, minimal_response());
3967 transport.shutdown();
3968 server.join().unwrap();
3969 }
3970
3971 #[test]
3972 fn segment_http_failures_have_named_codes_and_safe_cause_details() {
3973 for signed in [false, true] {
3974 for body_failure in [false, true] {
3975 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3976 let base = format!("http://{}", listener.local_addr().unwrap());
3977 let server = std::thread::spawn(move || {
3978 let (mut socket, _) = listener.accept().unwrap();
3979 let mut reader = BufReader::new(socket.try_clone().unwrap());
3980 loop {
3981 let mut line = String::new();
3982 reader.read_line(&mut line).unwrap();
3983 if line == "\r\n" {
3984 break;
3985 }
3986 assert!(!line.is_empty());
3987 }
3988 socket.write_all(if body_failure {
3989 b"HTTP/1.1 200 OK\r\nContent-Length: 999\r\nConnection: close\r\n\r\nshort"
3990 } else {
3991 b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
3992 }).unwrap();
3993 });
3994 let mut transport =
3995 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
3996 let mut progress = |_| {};
3997 let error = if signed {
3998 transport.fetch_url(
3999 &format!("{base}/signed?signature=secret"),
4000 1 << 20,
4001 &mut progress,
4002 )
4003 } else {
4004 transport.download_segment(
4005 &SegmentRequest {
4006 segment_id: "sha256:test".into(),
4007 table: "tasks".into(),
4008 requested_scopes_json: "{}".into(),
4009 byte_length: 1 << 20,
4010 },
4011 &mut progress,
4012 )
4013 }
4014 .unwrap_err();
4015 assert_eq!(error.code, "sync.transport_failed");
4016 assert_eq!(error.message, "segment transfer failed");
4017 let details = error.details.unwrap();
4018 assert!(details.get("path").is_none());
4019 assert!(details.get("causeMessage").is_none());
4020 assert_eq!(
4021 details["causeKind"],
4022 if body_failure { "body" } else { "status" }
4023 );
4024 assert_eq!(details["httpStatus"], if body_failure { 200 } else { 403 });
4025 server.join().unwrap();
4026 }
4027 }
4028 }
4029
4030 #[test]
4031 fn blob_status_classification_uses_the_actual_http_response() {
4032 for (status, expected) in [
4033 (404, "blob.not_found"),
4034 (401, "sync.auth_required"),
4035 (403, "blob.forbidden"),
4036 (500, "transport.failed"),
4037 ] {
4038 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4039 let base = format!("http://{}", listener.local_addr().unwrap());
4040 let server = std::thread::spawn(move || {
4041 let (mut socket, _) = listener.accept().unwrap();
4042 let mut reader = BufReader::new(socket.try_clone().unwrap());
4043 loop {
4044 let mut line = String::new();
4045 reader.read_line(&mut line).unwrap();
4046 if line == "\r\n" {
4047 break;
4048 }
4049 assert!(!line.is_empty());
4050 }
4051 let body = r#"{"code":"blob.not_found","message":"https://host/secret-path?secret-query#secret-fragment","retryable":false}"#;
4052 write!(socket, "HTTP/1.1 {status} Error\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();
4053 });
4054 let mut transport =
4055 HostTransport::from_config(&serde_json::json!({"baseUrl":base})).unwrap();
4056 let error = transport.blob_download("sha256:test", None).unwrap_err();
4057 assert_eq!(error.code, expected);
4058 assert_eq!(
4059 error.details,
4060 Some(serde_json::json!({
4061 "causeKind": "status",
4062 "httpStatus": status,
4063 "retryable": false
4064 }))
4065 );
4066 assert!(!error.message.contains("secret"));
4067 assert!(!error.message.contains("http"));
4068 server.join().unwrap();
4069 }
4070 }
4071
4072 #[test]
4073 fn refused_segment_connection_has_a_named_code_without_an_invented_status() {
4074 for signed in [false, true] {
4075 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4076 let base = format!("http://{}", listener.local_addr().unwrap());
4077 drop(listener);
4078 let mut transport =
4079 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
4080 let mut progress = |_| {};
4081 let error = if signed {
4082 transport.fetch_url(
4083 &format!("{base}/signed?signature=secret"),
4084 1 << 20,
4085 &mut progress,
4086 )
4087 } else {
4088 transport.download_segment(
4089 &SegmentRequest {
4090 segment_id: "sha256:test".into(),
4091 table: "tasks".into(),
4092 requested_scopes_json: "{}".into(),
4093 byte_length: 1 << 20,
4094 },
4095 &mut progress,
4096 )
4097 }
4098 .unwrap_err();
4099 assert_eq!(error.code, "sync.transport_failed");
4100 assert!(error.details.as_ref().unwrap().get("httpStatus").is_none());
4101 }
4102 }
4103
4104 #[test]
4105 fn segment_http_reports_intermediate_bytes_for_direct_and_signed_urls() {
4106 for signed in [false, true] {
4107 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4108 let base = format!("http://{}", listener.local_addr().unwrap());
4109 let server = std::thread::spawn(move || {
4110 let (mut socket, _) = listener.accept().unwrap();
4111 let mut reader = BufReader::new(socket.try_clone().unwrap());
4112 let mut headers = String::new();
4113 loop {
4114 let mut line = String::new();
4115 reader.read_line(&mut line).unwrap();
4116 if line == "\r\n" {
4117 break;
4118 }
4119 assert!(!line.is_empty());
4120 headers.push_str(&line.to_ascii_lowercase());
4121 }
4122 assert_eq!(headers.contains("authorization: bearer test"), !signed);
4123 assert_eq!(headers.contains("x-syncular-scopes:"), !signed);
4124 let bytes = vec![7; 128 * 1024 + 17];
4125 write!(
4126 socket,
4127 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
4128 bytes.len()
4129 )
4130 .unwrap();
4131 socket.write_all(&bytes).unwrap();
4132 });
4133 let mut transport = HostTransport::from_config(
4134 &serde_json::json!({"baseUrl": base, "headers": {"authorization": "Bearer test"}}),
4135 )
4136 .unwrap();
4137 let mut updates = Vec::new();
4138 let mut progress = |bytes| updates.push(bytes);
4139 let bytes = if signed {
4140 transport
4141 .fetch_url(&format!("{base}/signed"), 1 << 20, &mut progress)
4142 .unwrap()
4143 } else {
4144 transport
4145 .download_segment(
4146 &SegmentRequest {
4147 segment_id: "sha256:test".into(),
4148 table: "tasks".into(),
4149 requested_scopes_json: "{}".into(),
4150 byte_length: 1 << 20,
4151 },
4152 &mut progress,
4153 )
4154 .unwrap()
4155 };
4156 assert_eq!(bytes.len(), 128 * 1024 + 17);
4157 assert!(updates.iter().any(|n| *n > 0 && *n < bytes.len() as u64));
4158 assert_eq!(updates.last(), Some(&(bytes.len() as u64)));
4159 assert!(updates.windows(2).all(|pair| pair[0] < pair[1]));
4160 server.join().unwrap();
4161 }
4162 }
4163
4164 #[test]
4165 fn request_over_cap_is_refused_before_network() {
4166 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4167 listener.set_nonblocking(true).unwrap();
4168 let base = format!("http://{}", listener.local_addr().unwrap());
4169 let mut transport =
4170 HostTransport::from_config(&serde_json::json!({"baseUrl": base, "maxRequestBytes": 4}))
4171 .unwrap();
4172 let error = transport
4173 .blob_upload("sha256:test", &[0u8; 8], None)
4174 .unwrap_err();
4175 assert_eq!(error.code, "transport.request_too_large");
4176 assert_eq!(
4177 error.details,
4178 Some(serde_json::json!({ "limit": 4, "size": 8 }))
4179 );
4180 assert!(
4181 matches!(listener.accept(), Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock),
4182 "the request cap is checked before any connection"
4183 );
4184 }
4185
4186 #[test]
4187 fn credential_bearing_redirects_are_refused_without_contacting_the_destination() {
4188 for credentials in ["headers", "userinfo", "password"] {
4189 for redirects in ["deny", "follow"] {
4190 let destination = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4191 destination.set_nonblocking(true).unwrap();
4192 let location = format!("http://{}/landed", destination.local_addr().unwrap());
4193 let origin = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4194 let host = origin.local_addr().unwrap();
4195 let (base, headers) = match credentials {
4196 "headers" => (
4197 format!("http://{host}"),
4198 serde_json::json!({"authorization": "Bearer secret"}),
4199 ),
4200 "userinfo" => (format!("http://user:secret@{host}"), serde_json::json!({})),
4201 _ => (format!("http://:secret@{host}"), serde_json::json!({})),
4202 };
4203 let server = std::thread::spawn(move || {
4204 let (mut socket, _) = origin.accept().unwrap();
4205 let mut reader = BufReader::new(socket.try_clone().unwrap());
4206 loop {
4207 let mut line = String::new();
4208 reader.read_line(&mut line).unwrap();
4209 if line == "\r\n" {
4210 break;
4211 }
4212 }
4213 let _ = write!(
4214 socket,
4215 "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
4216 );
4217 });
4218 let mut transport = HostTransport::from_config(&serde_json::json!({
4219 "baseUrl": base,
4220 "redirects": redirects,
4221 "headers": headers,
4222 }))
4223 .unwrap();
4224 let error = transport.blob_download("sha256:test", None).unwrap_err();
4225 assert_eq!(
4226 error.code, "transport.redirect",
4227 "{credentials} {redirects}"
4228 );
4229 let details = error.details.unwrap();
4230 assert_eq!(details["causeKind"], "redirect");
4231 assert_eq!(details["httpStatus"], 302);
4232 assert!(
4233 matches!(destination.accept(), Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock),
4234 "{credentials} {redirects}: the destination is never contacted"
4235 );
4236 server.join().unwrap();
4237 }
4238 }
4239 }
4240
4241 #[test]
4242 fn websocket_handshake_redirects_obey_the_redirect_policy() {
4243 for credential in ["headers", "userinfo", "query", "none"] {
4250 let redirects = "deny";
4251 let destination = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4252 let destination_addr = destination.local_addr().unwrap();
4253 let location = format!("ws://{destination_addr}/realtime");
4254 let landed = Arc::new(Mutex::new(Vec::new()));
4258 let landed_writer = Arc::clone(&landed);
4259 let destination_thread = std::thread::spawn(move || loop {
4260 let (mut socket, _) = destination.accept().unwrap();
4261 let mut reader = BufReader::new(socket.try_clone().unwrap());
4262 let mut head = String::new();
4263 loop {
4264 let mut line = String::new();
4265 reader.read_line(&mut line).unwrap();
4266 if line == "\r\n" {
4267 break;
4268 }
4269 head.push_str(&line);
4270 }
4271 let sentinel = head.starts_with("GET /sentinel");
4272 landed_writer.lock().unwrap().push(head);
4273 let _ = write!(
4274 socket,
4275 "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
4276 );
4277 if sentinel {
4278 break;
4279 }
4280 });
4281 let origin = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4282 let origin_addr = origin.local_addr().unwrap();
4283 let (headers, ws_url) = match credential {
4284 "headers" => (serde_json::json!({"authorization": "Bearer secret"}), None),
4285 "userinfo" => (
4286 serde_json::json!({}),
4287 Some(format!("ws://user:secret@{origin_addr}/realtime")),
4288 ),
4289 "query" => (
4290 serde_json::json!({}),
4291 Some(format!("ws://{origin_addr}/realtime?token=secret")),
4292 ),
4293 _ => (serde_json::json!({}), None),
4294 };
4295 let (head_tx, head_rx) = std::sync::mpsc::channel();
4296 let origin_thread = std::thread::spawn(move || {
4297 let (mut socket, _) = origin.accept().unwrap();
4298 let mut reader = BufReader::new(socket.try_clone().unwrap());
4299 let mut head = String::new();
4300 loop {
4301 let mut line = String::new();
4302 reader.read_line(&mut line).unwrap();
4303 if line == "\r\n" {
4304 break;
4305 }
4306 head.push_str(&line);
4307 }
4308 let _ = head_tx.send(head);
4309 let _ = write!(
4310 socket,
4311 "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
4312 );
4313 });
4314 let mut config = serde_json::json!({
4315 "baseUrl": format!("http://{origin_addr}"),
4316 "headers": headers,
4317 "redirects": redirects,
4318 });
4319 if let Some(ws_url) = &ws_url {
4320 config["wsUrl"] = serde_json::json!(ws_url);
4321 }
4322 let mut transport = HostTransport::from_config(&config).unwrap();
4323 let result = transport.realtime_connect("c1");
4324 let mut sentinel = std::net::TcpStream::connect(destination_addr).unwrap();
4327 let _ = write!(sentinel, "GET /sentinel HTTP/1.1\r\nHost: sentinel\r\n\r\n");
4328 destination_thread.join().unwrap();
4329 let landed = landed.lock().unwrap().clone();
4330 origin_thread.join().unwrap();
4331 let error = result.unwrap_err();
4332 assert_eq!(
4333 error.code, "transport.redirect",
4334 "{credential}: a refused handshake redirect surfaces the redirect code"
4335 );
4336 let details = error.details.unwrap();
4337 assert_eq!(details["causeKind"], "redirect");
4338 assert_eq!(details["httpStatus"], 302);
4339 assert_eq!(
4340 landed.len(),
4341 1,
4342 "{credential}: the redirect target is never contacted: {landed:?}"
4343 );
4344 assert!(landed[0].starts_with("GET /sentinel"), "{landed:?}");
4345 let head = head_rx.recv().unwrap();
4346 if credential == "userinfo" {
4347 assert!(
4348 !head.contains("secret"),
4349 "the handshake request line drops URL userinfo"
4350 );
4351 }
4352 if credential == "query" {
4353 assert!(
4354 head.contains("token=secret"),
4355 "the handshake request line carries the realtime query"
4356 );
4357 }
4358 }
4359 }
4360
4361 const GZIP_BOMB: [u8; 548] = [
4364 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xed, 0xc9, 0xb1, 0x11, 0x80,
4365 0x20, 0x10, 0x00, 0xb0, 0x59, 0x1c, 0x80, 0xa1, 0xfe, 0x90, 0xb3, 0x41, 0xf0, 0x7c, 0x2d,
4366 0xd8, 0xde, 0x09, 0xdc, 0x20, 0x69, 0x93, 0x6b, 0xd4, 0xb7, 0xc7, 0x5d, 0xb2, 0x1d, 0x67,
4367 0x1b, 0x4f, 0xb9, 0x62, 0xf5, 0x19, 0xfb, 0x96, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4368 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4369 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4370 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4371 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4372 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4373 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4374 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4375 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4376 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4377 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4378 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4379 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4380 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4381 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4382 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4383 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4384 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4385 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4386 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4387 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4388 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4389 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4390 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4391 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4392 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4393 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4394 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4395 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4396 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4397 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4398 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
4399 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0xf8, 0x8b, 0x0f,
4400 0xdb, 0x82, 0xf4, 0xa9, 0x40, 0x0d, 0x03, 0x00,
4401 ];
4402
4403 #[test]
4404 fn decoded_response_cap_counts_decompressed_bytes() {
4405 for cap in [200000u64, 199999] {
4406 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4407 let base = format!("http://{}", listener.local_addr().unwrap());
4408 let server = std::thread::spawn(move || {
4409 let (mut socket, _) = listener.accept().unwrap();
4410 let mut reader = BufReader::new(socket.try_clone().unwrap());
4411 loop {
4412 let mut line = String::new();
4413 reader.read_line(&mut line).unwrap();
4414 if line == "\r\n" {
4415 break;
4416 }
4417 }
4418 let _ = write!(
4419 socket,
4420 "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
4421 GZIP_BOMB.len()
4422 );
4423 let _ = socket.write_all(&GZIP_BOMB);
4424 });
4425 let mut transport = HostTransport::from_config(
4426 &serde_json::json!({"baseUrl": base, "maxResponseBytes": cap}),
4427 )
4428 .unwrap();
4429 let result = transport.fetch_blob_url(&format!("{base}/signed"), None);
4430 if cap == 200000 {
4431 let body = result.expect("a cap equal to the decoded size admits the body");
4432 assert_eq!(body.len(), 200000);
4433 } else {
4434 let error = result.unwrap_err();
4435 assert_eq!(error.code, "transport.response_too_large");
4436 assert_eq!(error.details, Some(serde_json::json!({ "limit": cap })));
4437 }
4438 server.join().unwrap();
4439 }
4440 }
4441
4442 #[test]
4443 fn a_json_http_error_body_reaches_the_caller() {
4444 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
4445 let base = format!("http://{}", listener.local_addr().unwrap());
4446 let body = r#"{"code":"sync.invalid_client_id","category":"invalid-request","retryable":false,"recommendedAction":"resetClientId","message":"clientId is bound to a different actor in this partition (§1.5)","details":{"actorId":"actor-b"}}"#;
4447 let server = std::thread::spawn(move || {
4448 let (mut socket, _) = listener.accept().unwrap();
4449 let mut reader = BufReader::new(socket.try_clone().unwrap());
4450 loop {
4451 let mut line = String::new();
4452 reader.read_line(&mut line).unwrap();
4453 if line == "\r\n" {
4454 break;
4455 }
4456 }
4457 let _ = write!(
4458 socket,
4459 "HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
4460 body.len(),
4461 body
4462 );
4463 });
4464 let mut transport =
4465 HostTransport::from_config(&serde_json::json!({"baseUrl": base})).unwrap();
4466 let error = transport.sync(&[]).unwrap_err();
4467 assert_eq!(error.code, "sync.invalid_client_id");
4468 assert_eq!(
4469 error.message,
4470 "clientId is bound to a different actor in this partition (§1.5)"
4471 );
4472 let details = error.details.unwrap();
4473 assert_eq!(details["recommendedAction"], "resetClientId");
4474 assert_eq!(details["category"], "invalid-request");
4475 assert_eq!(details["retryable"], false);
4476 assert_eq!(details["httpStatus"], 400);
4477 assert_eq!(details["actorId"], "actor-b");
4478 server.join().unwrap();
4479 }
4480}
4481
4482#[cfg(test)]
4483mod local_authority_tests {
4484 use super::HostTransport;
4485 use crate::Transport;
4486
4487 #[cfg(not(feature = "local-authority"))]
4488 #[test]
4489 fn a_local_authority_path_without_the_feature_fails_with_the_unconfigured_code() {
4490 let error = HostTransport::from_config(
4491 &serde_json::json!({ "localAuthorityPath": "authority.sqlite" }),
4492 )
4493 .err()
4494 .expect("a build without local-authority refuses localAuthorityPath");
4495 assert!(
4496 error.starts_with("client.local_authority_unconfigured:"),
4497 "{error}"
4498 );
4499 }
4500
4501 #[test]
4502 fn a_config_without_a_local_authority_reports_none() {
4503 let mut transport = HostTransport::from_config(&serde_json::json!({}))
4504 .ok()
4505 .unwrap();
4506 assert_eq!(transport.local_authority_state(), None);
4507 let error = transport.attach_local_authority().unwrap_err();
4508 assert_eq!(error.code, "client.local_authority_unconfigured");
4509 let error = transport.abandon_local_authority_attach().unwrap_err();
4510 assert_eq!(error.code, "client.local_authority_unconfigured");
4511 }
4512
4513 #[cfg(feature = "local-authority")]
4514 mod with_feature {
4515 use serde_json::{json, Map, Value};
4516 use syncular_authority::LocalAuthority;
4517
4518 use super::super::{HostTransport, LocalAuthorityTransport};
4519 use crate::{
4520 ClientLimits, Mutation, SegmentFetch, SegmentRequest, SyncClient, SyncOutcome,
4521 Transport, TransportError, UnitScopes,
4522 };
4523
4524 fn schema() -> Value {
4525 json!({
4526 "version": 1,
4527 "tables": [{
4528 "name": "todos",
4529 "columns": [
4530 { "name": "id", "type": "string", "nullable": false },
4531 { "name": "list_id", "type": "string", "nullable": false }
4532 ],
4533 "primaryKey": "id",
4534 "scopes": [{ "pattern": "list:{list_id}", "column": "list_id" }]
4535 }]
4536 })
4537 }
4538
4539 struct AuthorityServer {
4542 server: LocalAuthority,
4543 syncs: usize,
4544 }
4545
4546 impl Transport for AuthorityServer {
4547 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
4548 self.syncs += 1;
4549 self.server.sync(request).map_err(|error| TransportError {
4550 code: error.code,
4551 message: error.message,
4552 details: error.details,
4553 })
4554 }
4555 fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
4556 Err(TransportError::new("sync.transport_failed", "no socket"))
4557 }
4558 fn download_segment(
4559 &mut self,
4560 _request: &SegmentRequest,
4561 _on_progress: &mut dyn FnMut(u64),
4562 ) -> Result<Vec<u8>, TransportError> {
4563 Err(TransportError::new("sync.transport_failed", "no segments"))
4564 }
4565 fn realtime_connect(&mut self, _client_id: &str) -> Result<(), TransportError> {
4566 Err(TransportError::new("sync.transport_failed", "no socket"))
4567 }
4568 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
4569 Ok(())
4570 }
4571 fn realtime_close(&mut self) -> Result<(), TransportError> {
4572 Ok(())
4573 }
4574 }
4575
4576 #[test]
4577 fn a_configured_path_routes_to_the_authority_and_refuses_what_it_lacks() {
4578 let mut transport =
4579 HostTransport::from_config(&json!({ "localAuthorityPath": ":memory:" }))
4580 .ok()
4581 .unwrap();
4582 assert!(matches!(transport, HostTransport::Local(_)));
4583 assert_eq!(transport.local_authority_state(), None);
4585 assert_eq!(
4586 transport.sync(b"").unwrap_err().code,
4587 "client.local_authority_unconfigured"
4588 );
4589 assert_eq!(
4590 transport.realtime_connect("device-1").unwrap_err().code,
4591 "sync.invalid_request"
4592 );
4593 let refused = transport.download_segments(
4594 &[SegmentFetch::Url {
4595 url: "https://cdn.example.com/s1".to_owned(),
4596 byte_length: 1 << 20,
4597 }],
4598 &mut |_, _| {},
4599 );
4600 assert_eq!(refused.len(), 1);
4601 assert_eq!(
4602 refused[0].as_ref().unwrap_err().code,
4603 "sync.invalid_request"
4604 );
4605 let error = transport.attach_local_authority().unwrap_err();
4607 assert_eq!(error.code, "client.local_authority_unconfigured");
4608 }
4609
4610 #[cfg(feature = "crdt-yjs")]
4613 #[test]
4614 fn a_crdt_commit_through_the_local_authority_is_merged() {
4615 let schema = json!({
4616 "version": 1,
4617 "tables": [{
4618 "name": "notes",
4619 "columns": [
4620 { "name": "id", "type": "string", "nullable": false },
4621 { "name": "list_id", "type": "string", "nullable": false },
4622 { "name": "doc", "type": "crdt", "nullable": true, "crdtType": "yjs-doc" }
4623 ],
4624 "primaryKey": "id",
4625 "scopes": [{ "pattern": "list:{list_id}", "column": "list_id" }]
4626 }]
4627 });
4628 let mut transport =
4629 HostTransport::from_config(&json!({ "localAuthorityPath": ":memory:" }))
4630 .ok()
4631 .unwrap();
4632 transport
4633 .open_local_authority(&schema, None)
4634 .expect("open the local authority");
4635 let mut client = SyncClient::new_with_identity(
4636 Some("device-1".to_owned()),
4637 &schema,
4638 ClientLimits::default(),
4639 )
4640 .expect("client");
4641 client
4642 .subscribe(UnitScopes::from([("list_id".to_owned(), "l1".to_owned())]))
4643 .expect("subscribe");
4644 client
4645 .mutate(vec![Mutation::Upsert {
4646 table: "notes".to_owned(),
4647 values: Map::from_iter([
4648 ("id".to_owned(), json!("n1")),
4649 ("list_id".to_owned(), json!("l1")),
4650 ("doc".to_owned(), Value::Null),
4651 ]),
4652 base_version: None,
4653 }])
4654 .expect("mutate");
4655 client
4656 .crdt_insert_text("notes", "n1", "doc", "text", 0, "éa")
4657 .expect("crdt insert");
4658 let outcome = client.sync_until_idle(&mut transport, None);
4659 assert!(
4660 matches!(outcome, SyncOutcome::Ok(_)),
4661 "{:?}",
4662 outcome.to_json()
4663 );
4664 assert!(client.pending_commit_ids().is_empty());
4665 assert!(client.rejections().is_empty(), "{:?}", client.rejections());
4666 assert_eq!(
4667 client.crdt_text("notes", "n1", "doc", "text").unwrap(),
4668 "éa"
4669 );
4670 }
4671
4672 #[test]
4673 fn the_authority_path_must_differ_from_the_replica_and_be_a_string() {
4674 let mut transport =
4675 HostTransport::from_config(&json!({ "localAuthorityPath": "/data/app.sqlite" }))
4676 .ok()
4677 .unwrap();
4678 let error = transport
4679 .open_local_authority(&schema(), Some("/data/app.sqlite"))
4680 .unwrap_err();
4681 assert_eq!(error.code, "sync.invalid_request");
4682 for path in [json!(""), json!(7)] {
4683 assert!(
4684 HostTransport::from_config(&json!({ "localAuthorityPath": path })).is_err()
4685 );
4686 }
4687 }
4688
4689 #[test]
4690 fn rounds_reach_the_authority_until_attach_then_the_remote_server() {
4691 let schema = schema();
4692 let remote = AuthorityServer {
4693 server: LocalAuthority::open(":memory:", &schema).expect("open the remote"),
4694 syncs: 0,
4695 };
4696 let mut transport = LocalAuthorityTransport::new(":memory:", remote, true);
4697 transport
4698 .open_local_authority(&schema, None)
4699 .expect("open the local authority");
4700 assert_eq!(transport.local_authority_state(), Some("local"));
4701
4702 let mut client = SyncClient::new_with_identity(
4703 Some("device-1".to_owned()),
4704 &schema,
4705 ClientLimits::default(),
4706 )
4707 .expect("client");
4708 client
4709 .subscribe(UnitScopes::from([("list_id".to_owned(), "l1".to_owned())]))
4710 .expect("subscribe");
4711 client
4712 .mutate(vec![Mutation::Upsert {
4713 table: "todos".to_owned(),
4714 values: Map::from_iter([
4715 ("id".to_owned(), json!("t1")),
4716 ("list_id".to_owned(), json!("l1")),
4717 ]),
4718 base_version: None,
4719 }])
4720 .expect("mutate");
4721 let outcome = client.sync_until_idle(&mut transport, None);
4722 assert!(
4723 matches!(outcome, SyncOutcome::Ok(_)),
4724 "{:?}",
4725 outcome.to_json()
4726 );
4727 assert!(client.pending_commit_ids().is_empty());
4728 assert_eq!(
4729 transport.remote.syncs, 0,
4730 "the local authority served the round"
4731 );
4732 let error = client.connect_realtime(&mut transport).unwrap_err();
4733 assert!(error.starts_with("sync.invalid_request:"), "{error}");
4734
4735 transport.attach_local_authority().expect("attach");
4736 assert_eq!(transport.local_authority_state(), Some("attached"));
4737 let uploads = transport.remote.syncs;
4738 assert!(uploads > 0, "attach uploaded through the remote transport");
4739
4740 let outcome = client.sync_until_idle(&mut transport, None);
4743 assert!(
4744 matches!(outcome, SyncOutcome::Ok(_)),
4745 "{:?}",
4746 outcome.to_json()
4747 );
4748 assert!(transport.remote.syncs > uploads);
4749 let rows = client.read_rows("todos").expect("rows");
4750 assert_eq!(rows.len(), 1);
4751 assert_eq!(rows[0].row_id, "t1");
4752 assert!(rows[0].version >= 1);
4753 }
4754 }
4755}