1use std::sync::{Arc, Mutex};
20use std::time::Duration;
21
22use crate::{
23 BlobDownload, BlobUploadGrant, SegmentFetch, SegmentRequest, Transport, TransportError,
24};
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
28pub enum RedirectPolicy {
29 #[default]
34 Deny,
35 Follow,
45}
46
47#[derive(Debug, Clone, Default)]
50pub struct HostTransportPolicy {
51 pub request_timeout: Option<Duration>,
53 pub round_deadline: Option<Duration>,
58 pub max_request_bytes: Option<u64>,
61 pub max_response_bytes: Option<u64>,
64 pub redirects: RedirectPolicy,
65}
66
67impl HostTransportPolicy {
68 pub fn validate(&self) -> Result<(), String> {
72 let now = std::time::Instant::now();
73 if let Some(timeout) = self.request_timeout {
74 if timeout.is_zero() || now.checked_add(timeout).is_none() {
75 return Err(
76 "sync.invalid_request: transport config requestTimeoutMs is out of range"
77 .to_owned(),
78 );
79 }
80 }
81 if let Some(deadline) = self.round_deadline {
82 if deadline.is_zero() || now.checked_add(deadline).is_none() {
83 return Err(
84 "sync.invalid_request: transport config roundDeadlineMs is out of range"
85 .to_owned(),
86 );
87 }
88 }
89 if self.max_request_bytes == Some(0) {
90 return Err(
91 "sync.invalid_request: transport config maxRequestBytes must be a positive integer"
92 .to_owned(),
93 );
94 }
95 if self.max_response_bytes == Some(0) {
96 return Err("sync.invalid_request: transport config maxResponseBytes must be a positive integer".to_owned());
97 }
98 Ok(())
99 }
100
101 pub fn from_config(config: &serde_json::Value) -> Result<Self, String> {
104 let bound = |key: &str, message: &'static str| -> Result<Option<u64>, String> {
105 match config.get(key) {
106 None | Some(serde_json::Value::Null) => Ok(None),
107 Some(value) => match value.as_u64() {
108 Some(number) if number >= 1 => Ok(Some(number)),
109 _ => Err(message.to_owned()),
110 },
111 }
112 };
113 let redirects = match config.get("redirects") {
114 None | Some(serde_json::Value::Null) => RedirectPolicy::Deny,
115 Some(serde_json::Value::String(value)) if value == "deny" => RedirectPolicy::Deny,
116 Some(serde_json::Value::String(value)) if value == "follow" => RedirectPolicy::Follow,
117 Some(_) => return Err(
118 "sync.invalid_request: transport config redirects must be \"deny\" or \"follow\""
119 .to_owned(),
120 ),
121 };
122 let policy = Self {
123 request_timeout: bound(
124 "requestTimeoutMs",
125 "sync.invalid_request: transport config requestTimeoutMs must be a positive integer",
126 )?
127 .map(Duration::from_millis),
128 round_deadline: bound(
129 "roundDeadlineMs",
130 "sync.invalid_request: transport config roundDeadlineMs must be a positive integer",
131 )?
132 .map(Duration::from_millis),
133 max_request_bytes: bound(
134 "maxRequestBytes",
135 "sync.invalid_request: transport config maxRequestBytes must be a positive integer",
136 )?,
137 max_response_bytes: bound(
138 "maxResponseBytes",
139 "sync.invalid_request: transport config maxResponseBytes must be a positive integer",
140 )?,
141 redirects,
142 };
143 policy.validate()?;
144 Ok(policy)
145 }
146}
147
148pub enum Inbound {
150 Text(String),
151 Binary(Vec<u8>),
152 Disconnected(TransportError),
155}
156
157pub struct InboundBuffer {
161 frames: Mutex<Vec<Inbound>>,
162 notify: Option<Arc<dyn Fn() + Send + Sync>>,
163}
164
165impl Default for InboundBuffer {
166 fn default() -> Self {
167 Self {
168 frames: Mutex::new(Vec::new()),
169 notify: None,
170 }
171 }
172}
173
174impl InboundBuffer {
175 pub fn with_notify(notify: Arc<dyn Fn() + Send + Sync>) -> Self {
176 Self {
177 frames: Mutex::new(Vec::new()),
178 notify: Some(notify),
179 }
180 }
181
182 pub fn push(&self, frame: Inbound) {
183 self.frames.lock().expect("inbound lock").push(frame);
184 if let Some(notify) = &self.notify {
185 notify();
186 }
187 }
188 fn take(&self) -> Vec<Inbound> {
189 std::mem::take(&mut *self.frames.lock().expect("inbound lock"))
190 }
191}
192
193#[allow(clippy::large_enum_variant)]
196pub enum HostTransport {
197 Null {
199 signed_urls: bool,
200 inbound: Arc<InboundBuffer>,
201 },
202 #[cfg(feature = "native-transport")]
203 Native(native::NativeTransport),
204}
205
206impl HostTransport {
207 pub fn from_config(config: &serde_json::Value) -> Result<Self, String> {
210 Self::new_from_config(config)
211 }
212
213 pub fn new_from_config(config: &serde_json::Value) -> Result<Self, String> {
216 Self::from_config_with_notify(config, None)
217 }
218
219 pub fn from_config_with_notify(
223 config: &serde_json::Value,
224 notify: Option<Arc<dyn Fn() + Send + Sync>>,
225 ) -> Result<Self, String> {
226 let policy = HostTransportPolicy::from_config(config)?;
227 Self::from_config_with_policy(config, policy, notify)
228 }
229
230 pub fn from_config_with_policy(
233 config: &serde_json::Value,
234 policy: HostTransportPolicy,
235 notify: Option<Arc<dyn Fn() + Send + Sync>>,
236 ) -> Result<Self, String> {
237 #[cfg(feature = "native-transport")]
238 {
239 if let Some(base_url) = config.get("baseUrl").and_then(|v| v.as_str()) {
240 return Ok(HostTransport::Native(native::NativeTransport::with_policy(
241 base_url, config, policy, notify,
242 )?));
243 }
244 }
245 #[cfg(not(feature = "native-transport"))]
246 {
247 policy.validate()?;
248 if config.get("baseUrl").is_some() {
249 return Err(
250 "this build has no native transport (rebuild with --features native-transport)"
251 .to_owned(),
252 );
253 }
254 }
255 Ok(HostTransport::Null {
256 signed_urls: false,
257 inbound: Arc::new(match notify {
258 Some(notify) => InboundBuffer::with_notify(notify),
259 None => InboundBuffer::default(),
260 }),
261 })
262 }
263
264 pub fn fork_round(&self) -> Self {
267 match self {
268 Self::Null {
269 signed_urls,
270 inbound,
271 } => Self::Null {
272 signed_urls: *signed_urls,
273 inbound: inbound.clone(),
274 },
275 #[cfg(feature = "native-transport")]
276 Self::Native(t) => Self::Native(t.fork_round()),
277 }
278 }
279
280 pub fn set_signed_urls(&mut self, value: bool) {
281 match self {
282 HostTransport::Null { signed_urls, .. } => *signed_urls = value,
283 #[cfg(feature = "native-transport")]
284 HostTransport::Native(t) => t.signed_urls = value,
285 }
286 }
287
288 pub fn set_headers(&mut self, headers: Vec<(String, String)>) {
291 match self {
292 HostTransport::Null { .. } => drop(headers),
293 #[cfg(feature = "native-transport")]
294 HostTransport::Native(t) => t.set_headers(headers),
295 }
296 }
297
298 pub fn set_policy(&mut self, policy: HostTransportPolicy) -> Result<(), String> {
303 policy.validate()?;
304 match self {
305 HostTransport::Null { .. } => {}
306 #[cfg(feature = "native-transport")]
307 HostTransport::Native(t) => t.set_policy(policy)?,
308 }
309 Ok(())
310 }
311
312 pub fn take_inbound(&mut self) -> Vec<Inbound> {
314 match self {
315 HostTransport::Null { inbound, .. } => inbound.take(),
316 #[cfg(feature = "native-transport")]
317 HostTransport::Native(t) => t.inbound.take(),
318 }
319 }
320
321 pub fn shutdown(&mut self) {
323 match self {
324 HostTransport::Null { .. } => {}
325 #[cfg(feature = "native-transport")]
326 HostTransport::Native(t) => t.shutdown(),
327 }
328 }
329}
330
331fn unavailable(op: &str) -> TransportError {
332 TransportError::new(
333 "transport.unavailable",
334 format!("{op} needs the native transport (build with --features native-transport)"),
335 )
336}
337
338#[cfg_attr(not(feature = "native-transport"), allow(unused_variables))]
341impl Transport for HostTransport {
342 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
343 match self {
344 HostTransport::Null { .. } => Err(unavailable("sync")),
345 #[cfg(feature = "native-transport")]
346 HostTransport::Native(t) => t.sync(request),
347 }
348 }
349
350 fn remote_operation(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
351 match self {
352 HostTransport::Null { .. } => Err(unavailable("remoteOperation")),
353 #[cfg(feature = "native-transport")]
354 HostTransport::Native(t) => t.remote_operation(request),
355 }
356 }
357
358 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
359 match self {
360 HostTransport::Null { .. } => Err(unavailable("realtimeSync")),
361 #[cfg(feature = "native-transport")]
362 HostTransport::Native(t) => t.realtime_sync(request),
363 }
364 }
365
366 fn download_segment(
367 &mut self,
368 request: &SegmentRequest,
369 on_progress: &mut dyn FnMut(u64),
370 ) -> Result<Vec<u8>, TransportError> {
371 match self {
372 HostTransport::Null { .. } => Err(unavailable("downloadSegment")),
373 #[cfg(feature = "native-transport")]
374 HostTransport::Native(t) => t.download_segment(request, on_progress),
375 }
376 }
377
378 fn supports_url_fetch(&self) -> bool {
379 match self {
380 HostTransport::Null { signed_urls, .. } => *signed_urls,
381 #[cfg(feature = "native-transport")]
382 HostTransport::Native(t) => t.signed_urls,
383 }
384 }
385
386 fn download_segments(
387 &mut self,
388 fetches: &[SegmentFetch],
389 on_progress: &mut (dyn FnMut(usize, u64) + Send),
390 ) -> Vec<Result<Vec<u8>, TransportError>> {
391 match self {
392 HostTransport::Null { .. } => fetches
393 .first()
394 .map(|_| Err(unavailable("downloadSegments")))
395 .into_iter()
396 .collect(),
397 #[cfg(feature = "native-transport")]
398 HostTransport::Native(t) => t.download_segments(fetches, on_progress),
399 }
400 }
401
402 fn fetch_url(
403 &mut self,
404 url: &str,
405 on_progress: &mut dyn FnMut(u64),
406 ) -> Result<Vec<u8>, TransportError> {
407 match self {
408 HostTransport::Null { .. } => Err(unavailable("fetchUrl")),
409 #[cfg(feature = "native-transport")]
410 HostTransport::Native(t) => t.fetch_url(url, on_progress),
411 }
412 }
413
414 fn blob_upload(
415 &mut self,
416 blob_id: &str,
417 bytes: &[u8],
418 media_type: Option<&str>,
419 ) -> Result<(), TransportError> {
420 match self {
421 HostTransport::Null { .. } => Err(unavailable("blobUpload")),
422 #[cfg(feature = "native-transport")]
423 HostTransport::Native(t) => t.blob_upload(blob_id, bytes, media_type),
424 }
425 }
426
427 fn blob_download(&mut self, blob_id: &str) -> Result<BlobDownload, TransportError> {
428 match self {
429 HostTransport::Null { .. } => Err(unavailable("blobDownload")),
430 #[cfg(feature = "native-transport")]
431 HostTransport::Native(t) => t.blob_download(blob_id),
432 }
433 }
434
435 fn fetch_blob_url(&mut self, url: &str) -> Result<Vec<u8>, TransportError> {
436 match self {
437 HostTransport::Null { .. } => Err(unavailable("fetchBlobUrl")),
438 #[cfg(feature = "native-transport")]
439 HostTransport::Native(t) => t.fetch_blob_url(url),
440 }
441 }
442
443 fn blob_upload_grant(
444 &mut self,
445 blob_id: &str,
446 byte_length: u64,
447 media_type: Option<&str>,
448 ) -> Result<BlobUploadGrant, TransportError> {
449 match self {
450 HostTransport::Null { .. } => Ok(BlobUploadGrant::None),
453 #[cfg(feature = "native-transport")]
454 HostTransport::Native(t) => t.blob_upload_grant(blob_id, byte_length, media_type),
455 }
456 }
457
458 fn blob_put_url(
459 &mut self,
460 url: &str,
461 bytes: &[u8],
462 media_type: Option<&str>,
463 ) -> Result<(), TransportError> {
464 match self {
465 HostTransport::Null { .. } => Err(unavailable("blobPutUrl")),
466 #[cfg(feature = "native-transport")]
467 HostTransport::Native(t) => t.blob_put_url(url, bytes, media_type),
468 }
469 }
470
471 fn realtime_connect(&mut self) -> Result<(), TransportError> {
472 match self {
473 HostTransport::Null { .. } => Err(unavailable("realtimeConnect")),
474 #[cfg(feature = "native-transport")]
475 HostTransport::Native(t) => {
476 t.set_realtime_client_id(None);
477 t.realtime_connect()
478 }
479 }
480 }
481
482 fn realtime_connect_for_client(&mut self, client_id: &str) -> Result<(), TransportError> {
483 match self {
484 HostTransport::Null { .. } => Err(unavailable("realtimeConnect")),
485 #[cfg(feature = "native-transport")]
486 HostTransport::Native(t) => {
487 t.set_realtime_client_id(Some(client_id));
488 t.realtime_connect()
489 }
490 }
491 }
492
493 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
494 match self {
495 HostTransport::Null { .. } => Err(unavailable("realtimeSend")),
496 #[cfg(feature = "native-transport")]
497 HostTransport::Native(t) => t.realtime_send(text),
498 }
499 }
500
501 fn realtime_close(&mut self) -> Result<(), TransportError> {
502 match self {
503 HostTransport::Null { .. } => Ok(()),
504 #[cfg(feature = "native-transport")]
505 HostTransport::Native(t) => t.realtime_close(),
506 }
507 }
508
509 fn round_deadline(&self) -> Option<std::time::Duration> {
510 match self {
511 HostTransport::Null { .. } => None,
512 #[cfg(feature = "native-transport")]
513 HostTransport::Native(t) => t.round_deadline(),
514 }
515 }
516
517 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
518 match self {
519 HostTransport::Null { .. } => {}
520 #[cfg(feature = "native-transport")]
521 HostTransport::Native(t) => t.set_round_deadline(deadline),
522 }
523 }
524}
525
526#[cfg(feature = "native-transport")]
527mod native {
528 use std::io::{self, Read, Write};
540 use std::net::{TcpStream, ToSocketAddrs};
541 use std::sync::atomic::{AtomicBool, Ordering};
542 use std::sync::{mpsc, Arc, Condvar, Mutex};
543 use std::thread::JoinHandle;
544 use std::time::{Duration, Instant};
545
546 use polling::{Event, Events, Poller};
547 use tungstenite::stream::MaybeTlsStream;
548 use tungstenite::Message;
549 use ureq::unversioned::resolver::DefaultResolver;
550 use ureq::unversioned::transport::{
551 time::Duration as UreqDuration, ConnectionDetails, Connector, DefaultConnector,
552 NextTimeout, Transport as UreqTransport,
553 };
554 use ureq::Timeout as UreqTimeout;
555
556 use super::{HostTransportPolicy, Inbound, InboundBuffer, RedirectPolicy};
557 use crate::{
558 BlobDownload, BlobUploadGrant, RealtimeRound, RoundInbound, SegmentFetch, SegmentRequest,
559 Transport, TransportError,
560 };
561
562 struct SocketRegistration {
564 poller: Arc<Poller>,
565 stream: TcpStream,
566 }
567
568 impl Drop for SocketRegistration {
569 fn drop(&mut self) {
570 let _ = self.poller.delete(&self.stream);
571 }
572 }
573
574 #[derive(Clone)]
575 struct SocketSender {
576 queue: mpsc::SyncSender<Outgoing>,
577 poller: Arc<Poller>,
578 }
579
580 struct Outgoing {
581 message: Message,
582 completed: mpsc::Sender<Result<(), TransportError>>,
583 }
584
585 const ROUND_IDLE_TIMEOUT: Duration = Duration::from_secs(30);
591 const REQUEST_CHUNK_BYTES: usize = 64 * 1024;
594
595 fn round_expiry(
598 last_progress: Instant,
599 idle: Duration,
600 ceiling: Option<Instant>,
601 ) -> (Instant, bool) {
602 let silent_at = last_progress + idle;
603 match ceiling {
604 Some(ceiling) if ceiling < silent_at => (ceiling, true),
605 _ => (silent_at, false),
606 }
607 }
608 #[derive(Default)]
615 pub(super) struct RoundChannel {
616 state: Mutex<RoundState>,
617 ready: Condvar,
618 }
619
620 #[derive(Default)]
621 struct RoundState {
622 round: RealtimeRound,
623 last_progress: Option<Instant>,
626 lost: bool,
628 outcome: Option<Result<Vec<u8>, TransportError>>,
630 }
631
632 impl RoundChannel {
633 fn begin(&self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
637 let mut state = self.state.lock().expect("round lock");
638 state.outcome = None;
639 let framed = state.round.begin(request)?;
640 state.last_progress = Some(Instant::now());
641 Ok(framed)
642 }
643
644 fn progressed(&self) {
646 self.state.lock().expect("round lock").last_progress = Some(Instant::now());
647 }
648
649 fn lost(&self) -> bool {
652 self.state.lock().expect("round lock").lost
653 }
654
655 fn route_binary(&self, frame: &[u8]) -> Option<Vec<u8>> {
659 let mut state = self.state.lock().expect("round lock");
660 match state.round.route_binary(frame) {
661 Ok(RoundInbound::Delta(body)) => Some(body),
662 Ok(RoundInbound::RoundProgress) => {
663 state.last_progress = Some(Instant::now());
664 None
665 }
666 Ok(RoundInbound::Ignored) => None,
667 Ok(RoundInbound::RoundComplete(bytes)) => {
668 state.outcome = Some(Ok(bytes));
669 self.ready.notify_all();
670 None
671 }
672 Err(error) => {
673 state.outcome = Some(Err(error));
674 self.ready.notify_all();
675 None
676 }
677 }
678 }
679
680 fn fail_in_flight(&self, error: TransportError) {
682 let mut state = self.state.lock().expect("round lock");
683 state.lost = true;
684 if state.round.in_flight() && state.outcome.is_none() {
685 state.round.abort();
686 state.outcome = Some(Err(error));
687 self.ready.notify_all();
688 }
689 }
690
691 fn wait(
694 &self,
695 ceiling: Option<Instant>,
696 idle: Duration,
697 ) -> Result<Vec<u8>, TransportError> {
698 let mut state = self.state.lock().expect("round lock");
699 while state.outcome.is_none() {
700 let (deadline, ceiling_binds) = round_expiry(
701 state.last_progress.unwrap_or_else(Instant::now),
702 idle,
703 ceiling,
704 );
705 let now = Instant::now();
706 if now >= deadline {
707 state.round.abort();
708 return Err(if ceiling_binds {
709 timeout_error("sync round deadline exceeded", "round_deadline")
710 } else {
711 TransportError::new(
712 "sync.transport_failed",
713 "realtime sync round made no progress (§8.7)",
714 )
715 });
716 }
717 let (guard, _timeout) = self
718 .ready
719 .wait_timeout(state, deadline - now)
720 .expect("round wait");
721 state = guard;
722 }
723 state.outcome.take().expect("outcome present")
724 }
725 }
726
727 fn transfer_error(
728 message: &'static str,
729 cause_kind: &'static str,
730 http_status: Option<u16>,
731 ) -> TransportError {
732 let mut details = serde_json::json!({ "causeKind": cause_kind });
733 if let Some(status) = http_status {
734 details["httpStatus"] = status.into();
735 }
736 let mut error = TransportError::new("transport.failed", message);
737 error.details = Some(details);
738 error
739 }
740
741 fn timeout_error(message: &'static str, cause_kind: &'static str) -> TransportError {
744 let mut error = TransportError::new("transport.timeout", message);
745 error.details = Some(serde_json::json!({ "causeKind": cause_kind }));
746 error
747 }
748
749 fn redirect_error(message: &'static str, http_status: Option<u16>) -> TransportError {
751 let mut details = serde_json::json!({ "causeKind": "redirect" });
752 if let Some(status) = http_status {
753 details["httpStatus"] = status.into();
754 }
755 let mut error = TransportError::new("transport.redirect", message);
756 error.details = Some(details);
757 error
758 }
759
760 fn redirect_refused(resp: &ureq::http::Response<ureq::Body>) -> Result<(), TransportError> {
765 let status = resp.status().as_u16();
766 if (300..400).contains(&status) {
767 return Err(redirect_error("redirect refused", Some(status)));
768 }
769 Ok(())
770 }
771
772 fn is_timeout_io(error: &std::io::Error) -> bool {
778 if error.kind() == std::io::ErrorKind::TimedOut {
779 return true;
780 }
781 let Some(inner) = error.get_ref() else {
782 return false;
783 };
784 let Some(ureq_error) = inner.downcast_ref::<ureq::Error>() else {
785 return false;
786 };
787 match ureq_error {
788 ureq::Error::Timeout(_) => true,
789 ureq::Error::Io(nested) => is_timeout_io(nested),
790 ureq::Error::Decompress(_, nested) => is_timeout_io(nested),
791 _ => false,
792 }
793 }
794
795 fn http_err(message: &'static str, error: ureq::Error) -> TransportError {
796 let (kind, status) = match error {
797 ureq::Error::StatusCode(status) if (300..400).contains(&status) => {
798 return redirect_error(message, Some(status))
799 }
800 ureq::Error::StatusCode(status) => ("status", Some(status)),
801 ureq::Error::Timeout(_) => return timeout_error(message, "timeout"),
802 ureq::Error::Io(error) => return io_err(message, error),
803 ureq::Error::HostNotFound
804 | ureq::Error::ConnectionFailed
805 | ureq::Error::ConnectProxyFailed(_) => ("connect", None),
806 ureq::Error::Tls(_)
807 | ureq::Error::Rustls(_)
808 | ureq::Error::Pem(_)
809 | ureq::Error::TlsRequired => ("tls", None),
810 ureq::Error::BadUri(_)
811 | ureq::Error::Http(_)
812 | ureq::Error::RequireHttpsOnly(_)
813 | ureq::Error::InvalidProxyUrl => ("request", None),
814 ureq::Error::Protocol(_) => ("protocol", None),
815 ureq::Error::BodyExceedsLimit(_)
816 | ureq::Error::BodyStalled
817 | ureq::Error::Decompress(_, _) => ("body", None),
818 ureq::Error::RedirectFailed | ureq::Error::TooManyRedirects => {
819 return redirect_error(message, None)
820 }
821 _ => ("unknown", None),
822 };
823 transfer_error(message, kind, status)
824 }
825
826 fn io_err(message: &'static str, error: std::io::Error) -> TransportError {
827 if is_timeout_io(&error) {
828 return timeout_error(message, "timeout");
829 }
830 let kind = match error.kind() {
831 std::io::ErrorKind::ConnectionRefused
832 | std::io::ErrorKind::ConnectionAborted
833 | std::io::ErrorKind::ConnectionReset
834 | std::io::ErrorKind::NotConnected => "connect",
835 _ => "io",
836 };
837 transfer_error(message, kind, None)
838 }
839
840 fn ws_err(message: &'static str, error: tungstenite::Error) -> TransportError {
841 match error {
842 tungstenite::Error::Io(error) => io_err(message, error),
843 tungstenite::Error::Tls(_) => transfer_error(message, "tls", None),
844 tungstenite::Error::Http(response) if response.status().is_redirection() => {
847 redirect_error(message, Some(response.status().as_u16()))
848 }
849 tungstenite::Error::Http(response) => {
850 transfer_error(message, "status", Some(response.status().as_u16()))
851 }
852 tungstenite::Error::Url(_) | tungstenite::Error::HttpFormat(_) => {
853 transfer_error(message, "request", None)
854 }
855 _ => transfer_error(message, "protocol", None),
856 }
857 }
858
859 const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
860
861 fn remaining(deadline: Instant) -> io::Result<Duration> {
862 deadline
863 .checked_duration_since(Instant::now())
864 .filter(|left| !left.is_zero())
865 .ok_or_else(|| {
866 io::Error::new(
867 io::ErrorKind::TimedOut,
868 "websocket establishment deadline exceeded",
869 )
870 })
871 }
872
873 struct EstablishingStream {
874 socket: TcpStream,
875 deadline: Option<Instant>,
876 }
877
878 impl EstablishingStream {
879 fn timeout(&self) -> io::Result<Option<Duration>> {
880 self.deadline.map(remaining).transpose()
881 }
882 fn established(&mut self) -> io::Result<TcpStream> {
883 self.deadline = None;
884 self.socket.set_read_timeout(None)?;
885 self.socket.set_write_timeout(None)?;
886 self.socket.try_clone()
887 }
888 }
889
890 fn establishment_io<T>(result: io::Result<T>, establishing: bool) -> io::Result<T> {
891 result.map_err(|error| {
892 if establishing
893 && matches!(
894 error.kind(),
895 io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut
896 )
897 {
898 io::Error::new(
899 io::ErrorKind::TimedOut,
900 "websocket establishment deadline exceeded",
901 )
902 } else {
903 error
904 }
905 })
906 }
907
908 impl Read for EstablishingStream {
909 fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
910 if self.deadline.is_some() {
911 self.socket.set_read_timeout(self.timeout()?)?;
912 }
913 establishment_io(self.socket.read(bytes), self.deadline.is_some())
914 }
915 }
916 impl Write for EstablishingStream {
917 fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
918 if self.deadline.is_some() {
919 self.socket.set_write_timeout(self.timeout()?)?;
920 }
921 establishment_io(self.socket.write(bytes), self.deadline.is_some())
922 }
923 fn flush(&mut self) -> io::Result<()> {
924 if self.deadline.is_some() {
925 self.socket.set_write_timeout(self.timeout()?)?;
926 }
927 establishment_io(self.socket.flush(), self.deadline.is_some())
928 }
929 }
930
931 fn connect_websocket(
932 request: tungstenite::handshake::client::Request,
933 deadline: Instant,
934 ) -> Result<tungstenite::WebSocket<MaybeTlsStream<EstablishingStream>>, TransportError> {
935 let host = request
936 .uri()
937 .host()
938 .ok_or_else(|| transfer_error("realtime URL invalid", "request", None))?;
939 let host = host.trim_start_matches('[').trim_end_matches(']');
940 let port = match request.uri().scheme_str() {
941 Some("ws") => request.uri().port_u16().unwrap_or(80),
942 Some("wss") => request.uri().port_u16().unwrap_or(443),
943 _ => return Err(transfer_error("realtime URL invalid", "request", None)),
944 };
945 let host = host.to_owned();
948 let (resolved, addresses) = mpsc::sync_channel(1);
949 std::thread::Builder::new()
950 .name("syncular-dns".to_owned())
951 .spawn(move || {
952 let result = (host.as_str(), port)
953 .to_socket_addrs()
954 .map(|items| items.collect::<Vec<_>>());
955 let _ = resolved.send(result);
956 })
957 .map_err(|e| io_err("realtime DNS failed", e))?;
958 let addresses = addresses
959 .recv_timeout(remaining(deadline).map_err(|e| io_err("realtime DNS failed", e))?)
960 .map_err(|e| match e {
961 mpsc::RecvTimeoutError::Timeout => {
962 timeout_error("realtime DNS timed out", "timeout")
963 }
964 mpsc::RecvTimeoutError::Disconnected => {
965 transfer_error("realtime DNS failed", "io", None)
966 }
967 })?
968 .map_err(|e| io_err("realtime DNS failed", e))?;
969 let mut last_error = io::Error::new(io::ErrorKind::NotFound, "DNS returned no addresses");
970 let mut socket = None;
971 for address in addresses {
972 match TcpStream::connect_timeout(
973 &address,
974 remaining(deadline).map_err(|e| io_err("realtime TCP connect failed", e))?,
975 ) {
976 Ok(stream) => {
977 socket = Some(stream);
978 break;
979 }
980 Err(error) => last_error = error,
981 }
982 }
983 let socket = socket.ok_or_else(|| io_err("realtime TCP connect failed", last_error))?;
984 socket
985 .set_nodelay(true)
986 .map_err(|e| io_err("realtime TCP connect failed", e))?;
987 let stream = EstablishingStream {
988 socket,
989 deadline: Some(deadline),
990 };
991 let (ws, _) = tungstenite::client_tls_with_config(request, stream, None, None).map_err(
994 |e| match e {
995 tungstenite::HandshakeError::Failure(error) => {
996 ws_err("realtime connection failed", error)
997 }
998 tungstenite::HandshakeError::Interrupted(_) => {
1001 transfer_error("realtime connection failed", "protocol", None)
1002 }
1003 },
1004 )?;
1005 remaining(deadline).map_err(|e| io_err("realtime connection failed", e))?;
1006 Ok(ws)
1007 }
1008
1009 #[cfg(test)]
1010 mod establishment_tests {
1011 use super::*;
1012 use std::net::{Shutdown, TcpListener};
1013 use tungstenite::client::IntoClientRequest;
1014
1015 fn withheld_response(tls: bool, old: bool) {
1018 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1019 let url = format!(
1020 "{}://{}/realtime",
1021 if tls { "wss" } else { "ws" },
1022 listener.local_addr().unwrap()
1023 );
1024 let (accepted_tx, accepted_rx) = mpsc::channel();
1025 let (release_tx, release_rx) = mpsc::channel();
1026 let peer = std::thread::spawn(move || {
1027 let (mut stream, _) = listener.accept().unwrap();
1028 stream
1029 .set_read_timeout(Some(Duration::from_secs(2)))
1030 .unwrap();
1031 let mut bytes = [0; 4096];
1032 assert!(stream.read(&mut bytes).unwrap() > 0);
1033 accepted_tx.send(()).unwrap();
1034 release_rx.recv_timeout(Duration::from_secs(3)).unwrap();
1035 let _ = stream.shutdown(Shutdown::Both);
1036 });
1037 let (done_tx, done_rx) = mpsc::channel();
1038 let worker = std::thread::spawn(move || {
1039 let result = if old {
1040 tungstenite::connect(url)
1041 .map(|_| ())
1042 .map_err(|e| e.to_string())
1043 } else {
1044 connect_websocket(
1045 url.into_client_request().unwrap(),
1046 Instant::now() + Duration::from_millis(250),
1047 )
1048 .map(|_| ())
1049 .map_err(|e| format!("{}: {}", e.code, e.message))
1050 };
1051 done_tx.send(result).unwrap();
1052 });
1053 accepted_rx.recv_timeout(Duration::from_secs(2)).unwrap();
1054 if old {
1055 assert!(
1056 matches!(
1057 done_rx.recv_timeout(Duration::from_millis(500)),
1058 Err(mpsc::RecvTimeoutError::Timeout)
1059 ),
1060 "original connect unexpectedly had a deadline"
1061 );
1062 release_tx.send(()).unwrap();
1063 assert!(done_rx
1064 .recv_timeout(Duration::from_secs(2))
1065 .unwrap()
1066 .is_err());
1067 } else {
1068 let error = done_rx
1069 .recv_timeout(Duration::from_secs(2))
1070 .expect("establishment remained blocked")
1071 .unwrap_err();
1072 assert_eq!(error, "transport.timeout: realtime connection failed");
1073 eprintln!(
1074 "withheld {} produces {error}",
1075 if tls { "TLS" } else { "upgrade" }
1076 );
1077 release_tx.send(()).unwrap();
1078 }
1079 worker.join().unwrap();
1080 peer.join().unwrap();
1081 }
1082
1083 #[test]
1084 fn original_connect_blocks_on_withheld_upgrade_and_tls() {
1085 withheld_response(false, true);
1086 withheld_response(true, true);
1087 }
1088 #[test]
1089 fn establishment_deadline_errors_on_withheld_upgrade_and_tls() {
1090 withheld_response(false, false);
1091 withheld_response(true, false);
1092 }
1093
1094 #[test]
1095 fn slow_upgrade_bytes_cannot_renew_the_absolute_deadline() {
1096 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1097 let url = format!("ws://{}", listener.local_addr().unwrap());
1098 let peer = std::thread::spawn(move || {
1099 let (mut stream, _) = listener.accept().unwrap();
1100 let mut bytes = [0; 4096];
1101 let _ = stream.read(&mut bytes).unwrap();
1102 for byte in b"HTTP/1.1 101 Switching Protocols\r\n" {
1103 if stream.write_all(&[*byte]).is_err() {
1104 break;
1105 }
1106 std::thread::sleep(Duration::from_millis(40));
1107 }
1108 });
1109 let started = Instant::now();
1110 assert!(connect_websocket(
1111 url.into_client_request().unwrap(),
1112 started + Duration::from_millis(250)
1113 )
1114 .is_err());
1115 assert!(started.elapsed() < Duration::from_secs(1));
1116 peer.join().unwrap();
1117 }
1118
1119 #[test]
1120 fn saturated_tcp_accept_queue_returns_an_observable_error() {
1121 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1124 let address = listener.local_addr().unwrap();
1125 let mut queued = Vec::new();
1126 loop {
1127 match TcpStream::connect_timeout(&address, Duration::from_millis(50)) {
1128 Ok(stream) => queued.push(stream),
1129 Err(error) => {
1130 assert!(
1131 matches!(
1132 error.kind(),
1133 io::ErrorKind::TimedOut
1134 | io::ErrorKind::WouldBlock
1135 | io::ErrorKind::ConnectionRefused
1136 | io::ErrorKind::ConnectionReset
1137 ),
1138 "{error:?}"
1139 );
1140 break;
1141 }
1142 }
1143 assert!(queued.len() < 4096, "listener queue did not saturate");
1144 }
1145 let started = Instant::now();
1146 let url = format!("ws://{address}");
1147 let error = match connect_websocket(
1148 url.into_client_request().unwrap(),
1149 started + Duration::from_millis(250),
1150 ) {
1151 Err(error) => error,
1152 Ok(_) => panic!("full accept queue unexpectedly connected"),
1153 };
1154 assert!(
1155 (error.code == "transport.timeout" || error.code == "transport.failed")
1156 && (error.message == "realtime TCP connect failed"
1157 || error.message == "realtime connection failed"),
1158 "{}: {}",
1159 error.code,
1160 error.message
1161 );
1162 assert!(started.elapsed() < Duration::from_secs(2));
1163 }
1164 }
1165
1166 fn segment_error(mut error: TransportError) -> TransportError {
1170 if error.code == "transport.failed" {
1171 error.code = "sync.transport_failed".into();
1172 error.message = "segment transfer failed".into();
1173 }
1174 error
1175 }
1176
1177 fn is_would_block(e: &tungstenite::Error) -> bool {
1179 matches!(
1180 e,
1181 tungstenite::Error::Io(io) if matches!(
1182 io.kind(),
1183 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
1184 )
1185 )
1186 }
1187
1188 pub struct NativeTransport {
1189 base_url: String,
1190 ws_url: String,
1191 headers: Vec<(String, String)>,
1193 base_url_has_userinfo: bool,
1197 agent: ureq::Agent,
1198 policy: HostTransportPolicy,
1199 round_deadline_at: Option<std::time::Instant>,
1203 pub signed_urls: bool,
1204 pub inbound: Arc<InboundBuffer>,
1205 outgoing: Option<Box<SocketSender>>,
1207 reader: Option<JoinHandle<()>>,
1208 reader_stop: Arc<AtomicBool>,
1209 round: Arc<RoundChannel>,
1211 realtime_client_id: Option<String>,
1212 }
1213
1214 #[derive(Debug)]
1221 struct IdleConnector(Option<Duration>);
1222
1223 impl Connector<Box<dyn UreqTransport>> for IdleConnector {
1224 type Out = Box<dyn UreqTransport>;
1225
1226 fn connect(
1227 &self,
1228 _: &ConnectionDetails,
1229 chained: Option<Box<dyn UreqTransport>>,
1230 ) -> Result<Option<Self::Out>, ureq::Error> {
1231 Ok(chained.map(|inner| match self.0 {
1232 Some(idle) => IdleTransport { inner, idle }.boxed(),
1233 None => inner,
1234 }))
1235 }
1236 }
1237
1238 #[derive(Debug)]
1239 struct IdleTransport {
1240 inner: Box<dyn UreqTransport>,
1241 idle: Duration,
1242 }
1243
1244 fn idle_bound(timeout: NextTimeout, idle: Duration) -> NextTimeout {
1247 let idle = UreqDuration::Exact(idle);
1248 if timeout.after.is_not_happening() || timeout.after.as_nanos() > idle.as_nanos() {
1249 NextTimeout {
1250 after: idle,
1251 reason: UreqTimeout::RecvBody,
1252 }
1253 } else {
1254 timeout
1255 }
1256 }
1257
1258 impl UreqTransport for IdleTransport {
1259 fn buffers(&mut self) -> &mut dyn ureq::unversioned::transport::Buffers {
1260 self.inner.buffers()
1261 }
1262 fn transmit_output(
1263 &mut self,
1264 amount: usize,
1265 timeout: NextTimeout,
1266 ) -> Result<(), ureq::Error> {
1267 self.inner
1268 .transmit_output(amount, idle_bound(timeout, self.idle))
1269 }
1270 fn await_input(&mut self, timeout: NextTimeout) -> Result<bool, ureq::Error> {
1271 self.inner.await_input(idle_bound(timeout, self.idle))
1272 }
1273 fn is_open(&mut self) -> bool {
1274 self.inner.is_open()
1275 }
1276 fn is_tls(&self) -> bool {
1277 self.inner.is_tls()
1278 }
1279 }
1280
1281 fn build_agent(policy: &HostTransportPolicy) -> ureq::Agent {
1282 let config = ureq::Agent::config_builder()
1283 .timeout_resolve(policy.request_timeout)
1284 .timeout_connect(policy.request_timeout)
1285 .timeout_send_request(policy.request_timeout)
1286 .timeout_recv_response(policy.request_timeout)
1287 .max_redirects(match policy.redirects {
1288 RedirectPolicy::Deny => 0,
1289 RedirectPolicy::Follow => 10,
1290 })
1291 .max_redirects_will_error(matches!(policy.redirects, RedirectPolicy::Follow))
1292 .redirect_auth_headers(ureq::config::RedirectAuthHeaders::Never)
1293 .build();
1294 ureq::Agent::with_parts(
1295 config,
1296 DefaultConnector::new().chain(IdleConnector(policy.request_timeout)),
1297 DefaultResolver::default(),
1298 )
1299 }
1300
1301 fn derive_ws_url(base_url: &str) -> String {
1302 let ws = if let Some(rest) = base_url.strip_prefix("https://") {
1305 format!("wss://{rest}")
1306 } else if let Some(rest) = base_url.strip_prefix("http://") {
1307 format!("ws://{rest}")
1308 } else {
1309 base_url.to_owned()
1310 };
1311 let trimmed = ws.trim_end_matches('/');
1312 format!("{trimmed}/realtime")
1313 }
1314
1315 impl NativeTransport {
1316 pub fn fork_round(&self) -> Self {
1317 Self {
1318 base_url: self.base_url.clone(),
1319 ws_url: self.ws_url.clone(),
1320 headers: self.headers.clone(),
1321 base_url_has_userinfo: self.base_url_has_userinfo,
1322 agent: self.agent.clone(),
1323 policy: self.policy.clone(),
1324 round_deadline_at: None,
1325 signed_urls: self.signed_urls,
1326 inbound: self.inbound.clone(),
1327 outgoing: self.outgoing.clone(),
1328 reader: None,
1329 reader_stop: self.reader_stop.clone(),
1330 round: self.round.clone(),
1331 realtime_client_id: self.realtime_client_id.clone(),
1332 }
1333 }
1334
1335 pub fn new(
1336 base_url: &str,
1337 config: &serde_json::Value,
1338 notify: Option<Arc<dyn Fn() + Send + Sync>>,
1339 ) -> Result<Self, String> {
1340 let policy = HostTransportPolicy::from_config(config)?;
1341 Self::with_policy(base_url, config, policy, notify)
1342 }
1343
1344 pub fn with_policy(
1345 base_url: &str,
1346 config: &serde_json::Value,
1347 policy: HostTransportPolicy,
1348 notify: Option<Arc<dyn Fn() + Send + Sync>>,
1349 ) -> Result<Self, String> {
1350 policy.validate()?;
1351 let mut headers = Vec::new();
1352 if let Some(map) = config.get("headers").and_then(|v| v.as_object()) {
1353 for (k, v) in map {
1354 if let Some(s) = v.as_str() {
1355 headers.push((k.clone(), s.to_owned()));
1356 }
1357 }
1358 }
1359 let ws_url = config
1360 .get("wsUrl")
1361 .and_then(|v| v.as_str())
1362 .map(str::to_owned)
1363 .unwrap_or_else(|| derive_ws_url(base_url));
1364 let base_url_has_userinfo = url::Url::parse(base_url)
1367 .map(|url| !url.username().is_empty() || url.password().is_some())
1368 .unwrap_or(false);
1369 Ok(NativeTransport {
1370 base_url: base_url.trim_end_matches('/').to_owned(),
1371 ws_url,
1372 headers,
1373 base_url_has_userinfo,
1374 agent: build_agent(&policy),
1375 policy,
1376 round_deadline_at: None,
1377 signed_urls: false,
1378 inbound: Arc::new(match notify {
1379 Some(notify) => InboundBuffer::with_notify(notify),
1380 None => InboundBuffer::default(),
1381 }),
1382 outgoing: None,
1383 reader: None,
1384 reader_stop: Arc::new(AtomicBool::new(false)),
1385 round: Arc::new(RoundChannel::default()),
1386 realtime_client_id: None,
1387 })
1388 }
1389
1390 pub fn set_policy(&mut self, policy: HostTransportPolicy) -> Result<(), String> {
1394 policy.validate()?;
1395 self.agent = build_agent(&policy);
1396 self.policy = policy;
1397 Ok(())
1398 }
1399
1400 pub fn round_deadline(&self) -> Option<Duration> {
1401 self.policy.round_deadline
1402 }
1403
1404 pub fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
1405 self.round_deadline_at = deadline;
1406 }
1407
1408 fn round_remaining(&self) -> Result<Option<Duration>, TransportError> {
1413 match self.round_deadline_at {
1414 Some(deadline) => deadline
1415 .checked_duration_since(std::time::Instant::now())
1416 .map(Some)
1417 .ok_or_else(|| timeout_error("sync round deadline exceeded", "round_deadline")),
1418 None => Ok(None),
1419 }
1420 }
1421
1422 fn credentialed(&self) -> bool {
1425 !self.headers.is_empty() || self.base_url_has_userinfo
1426 }
1427
1428 fn finish_request<B>(
1434 &self,
1435 req: ureq::RequestBuilder<B>,
1436 credentialed: bool,
1437 ) -> Result<ureq::RequestBuilder<B>, TransportError> {
1438 let timeout = self.round_remaining()?;
1439 let follow = self.policy.redirects == RedirectPolicy::Follow && !credentialed;
1440 Ok(req
1441 .config()
1442 .timeout_global(timeout)
1443 .max_redirects(if follow { 10 } else { 0 })
1444 .max_redirects_will_error(true)
1445 .build())
1446 }
1447
1448 fn check_request_bytes(&self, len: u64) -> Result<(), TransportError> {
1450 match self.policy.max_request_bytes {
1451 Some(max) if len > max => {
1452 let mut error = TransportError::new(
1453 "transport.request_too_large",
1454 "request body exceeds the configured transport limit",
1455 );
1456 error.details = Some(serde_json::json!({ "limit": max, "size": len }));
1457 Err(error)
1458 }
1459 _ => Ok(()),
1460 }
1461 }
1462
1463 fn post_sync(&self, path: &str, body: &[u8]) -> Result<Vec<u8>, TransportError> {
1464 self.check_request_bytes(body.len() as u64)?;
1465 let credentialed = self.credentialed();
1466 let url = format!("{}{}", self.base_url, path);
1467 let mut req = self
1470 .agent
1471 .post(&url)
1472 .header("content-type", "application/vnd.syncular.sync.v2");
1473 for (k, v) in &self.headers {
1474 req = req.header(k.as_str(), v.as_str());
1475 }
1476 let req = self.finish_request(req, credentialed)?;
1477 let resp = req
1478 .send(body)
1479 .map_err(|e| http_err("sync request failed", e))?;
1480 read_body(resp, None, self.policy.max_response_bytes)
1481 }
1482
1483 fn post_operation(&self, body: &[u8]) -> Result<Vec<u8>, TransportError> {
1484 self.check_request_bytes(body.len() as u64)?;
1485 let credentialed = self.credentialed();
1486 let url = format!("{}/operations", self.base_url);
1487 let mut req = self.agent.post(&url).header(
1488 "content-type",
1489 "application/vnd.syncular.operations.v1+json",
1490 );
1491 for (key, value) in &self.headers {
1492 req = req.header(key.as_str(), value.as_str());
1493 }
1494 let req = self.finish_request(req, credentialed)?;
1495 let response = req
1496 .send(body)
1497 .map_err(|error| http_err("remote operation request failed", error))?;
1498 read_body(response, None, self.policy.max_response_bytes)
1499 }
1500
1501 fn get_bytes(&self, url: &str, with_headers: bool) -> Result<Vec<u8>, TransportError> {
1502 let mut req = self.agent.get(url);
1505 if with_headers {
1506 for (k, v) in &self.headers {
1507 req = req.header(k.as_str(), v.as_str());
1508 }
1509 }
1510 let req = self.finish_request(req, true)?;
1511 let resp = req.call().map_err(|e| http_err("URL fetch failed", e))?;
1512 read_body(resp, None, self.policy.max_response_bytes)
1513 }
1514
1515 pub fn set_headers(&mut self, headers: Vec<(String, String)>) {
1516 self.headers = headers;
1517 }
1518
1519 pub fn set_realtime_client_id(&mut self, client_id: Option<&str>) {
1520 self.realtime_client_id = client_id.map(str::to_owned);
1521 }
1522
1523 pub fn shutdown(&mut self) {
1524 self.reader_stop.store(true, Ordering::SeqCst);
1525 self.round.fail_in_flight(TransportError::new(
1528 "sync.transport_failed",
1529 "realtime disconnected mid-round (§8.7)",
1530 ));
1531 if let Some(outgoing) = &self.outgoing {
1532 let _ = outgoing.poller.notify();
1533 }
1534 if let Some(handle) = self.reader.take() {
1535 let _ = handle.join();
1536 }
1537 self.outgoing = None;
1538 }
1539
1540 fn send_message(&mut self, message: Message) -> Result<(), TransportError> {
1541 let (wait, round_bound) = match self.round_deadline_at {
1546 Some(deadline) => {
1547 match deadline.checked_duration_since(std::time::Instant::now()) {
1548 Some(remaining) if remaining < ROUND_IDLE_TIMEOUT => (remaining, true),
1549 Some(_) => (ROUND_IDLE_TIMEOUT, false),
1550 None => {
1553 return Err(timeout_error("realtime send timed out", "round_deadline"));
1554 }
1555 }
1556 }
1557 None => (ROUND_IDLE_TIMEOUT, false),
1558 };
1559 let result = (|| {
1560 let Some(outgoing) = &self.outgoing else {
1561 return Err(TransportError::new(
1562 "transport.failed",
1563 "realtime not connected",
1564 ));
1565 };
1566 let (completed, result) = mpsc::channel();
1567 outgoing
1568 .queue
1569 .try_send(Outgoing { message, completed })
1570 .map_err(|_| {
1571 TransportError::new("transport.failed", "realtime send queue unavailable")
1572 })?;
1573 outgoing
1574 .poller
1575 .notify()
1576 .map_err(|e| io_err("realtime wake failed", e))?;
1577 result.recv_timeout(wait).map_err(|_| {
1578 if round_bound {
1579 timeout_error("realtime send timed out", "round_deadline")
1580 } else {
1581 TransportError::new("transport.failed", "realtime send did not complete")
1582 }
1583 })?
1584 })();
1585 if let Err(error) = &result {
1588 self.shutdown();
1589 self.inbound.push(Inbound::Disconnected(error.clone()));
1592 }
1593 result
1594 }
1595 }
1596
1597 fn read_body(
1598 resp: ureq::http::Response<ureq::Body>,
1599 mut progress: Option<&mut dyn FnMut(u64)>,
1600 max: Option<u64>,
1601 ) -> Result<Vec<u8>, TransportError> {
1602 redirect_refused(&resp)?;
1603 use std::io::Read;
1604 let status = resp.status().as_u16();
1605 let mut reader = resp.into_body().into_with_config().reader();
1606 let mut bytes = Vec::new();
1607 let mut chunk = [0u8; 64 * 1024];
1608 let mut reported = 0;
1609 loop {
1610 let n = reader.read(&mut chunk).map_err(|error| {
1611 if is_timeout_io(&error) {
1612 timeout_error("response body read times out", "timeout")
1613 } else {
1614 transfer_error("response body read failed", "body", Some(status))
1615 }
1616 })?;
1617 if n == 0 {
1618 break;
1619 }
1620 if let Some(max) = max {
1623 if bytes.len() as u64 + n as u64 > max {
1624 return Err(response_too_large(max));
1625 }
1626 }
1627 bytes.extend_from_slice(&chunk[..n]);
1628 if bytes.len() - reported >= 64 * 1024 {
1629 if let Some(callback) = progress.as_mut() {
1630 callback(bytes.len() as u64);
1631 }
1632 reported = bytes.len();
1633 }
1634 }
1635 if bytes.len() != reported {
1636 if let Some(callback) = progress {
1637 callback(bytes.len() as u64);
1638 }
1639 }
1640 Ok(bytes)
1641 }
1642
1643 fn response_too_large(max: u64) -> TransportError {
1644 let mut error = TransportError::new(
1645 "transport.response_too_large",
1646 "response body exceeds the configured transport limit",
1647 );
1648 error.details = Some(serde_json::json!({ "limit": max }));
1649 error
1650 }
1651
1652 impl Transport for NativeTransport {
1653 fn sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1654 self.post_sync("/sync", request)
1655 }
1656
1657 fn remote_operation(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1658 self.post_operation(request)
1659 }
1660
1661 fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
1662 if self.outgoing.is_none() {
1672 return self.post_sync("/sync", request);
1673 }
1674 self.round_remaining()?;
1675 let framed = self.round.begin(request)?;
1676 let (tag, body) = framed.split_first().expect("framed request carries a tag");
1677 for chunk in body.chunks(REQUEST_CHUNK_BYTES) {
1678 let mut message = Vec::with_capacity(chunk.len() + 1);
1679 message.push(*tag);
1680 message.extend_from_slice(chunk);
1681 if let Err(error) = self.send_message(Message::Binary(message.into())) {
1682 self.round.fail_in_flight(error);
1683 break;
1684 }
1685 self.round.progressed();
1686 }
1687 let outcome = self.round.wait(self.round_deadline_at, ROUND_IDLE_TIMEOUT);
1688 if let Err(error) = &outcome {
1693 if !self.round.lost() {
1694 self.shutdown();
1695 self.inbound.push(Inbound::Disconnected(error.clone()));
1696 }
1697 }
1698 outcome
1699 }
1700
1701 fn download_segment(
1702 &mut self,
1703 request: &SegmentRequest,
1704 on_progress: &mut dyn FnMut(u64),
1705 ) -> Result<Vec<u8>, TransportError> {
1706 let url = format!("{}/segments/{}", self.base_url, request.segment_id);
1713 let credentialed = self.credentialed();
1714 let mut req = self
1715 .agent
1716 .get(&url)
1717 .header("x-syncular-scopes", &request.requested_scopes_json);
1718 for (k, v) in &self.headers {
1719 req = req.header(k.as_str(), v.as_str());
1720 }
1721 let req = self.finish_request(req, credentialed)?;
1722 let resp = req
1723 .call()
1724 .map_err(|error| segment_error(http_err("segment request failed", error)))?;
1725 read_body(resp, Some(on_progress), self.policy.max_response_bytes)
1726 .map_err(segment_error)
1727 }
1728
1729 fn supports_url_fetch(&self) -> bool {
1730 self.signed_urls
1731 }
1732
1733 fn download_segments(
1734 &mut self,
1735 fetches: &[SegmentFetch],
1736 on_progress: &mut (dyn FnMut(usize, u64) + Send),
1737 ) -> Vec<Result<Vec<u8>, TransportError>> {
1738 let workers = (0..fetches.len().min(4))
1747 .map(|_| {
1748 let mut worker = self.fork_round();
1749 worker.round_deadline_at = self.round_deadline_at;
1750 worker
1751 })
1752 .collect::<Vec<_>>();
1753 let slots = fetches.iter().map(|_| Mutex::new(None)).collect::<Vec<_>>();
1754 let progress = Mutex::new(on_progress);
1755 let next = std::sync::atomic::AtomicUsize::new(0);
1756 std::thread::scope(|scope| {
1757 for worker in workers {
1758 let (slots, progress, next) = (&slots, &progress, &next);
1759 scope.spawn(move || {
1760 let mut worker = worker;
1761 loop {
1762 let index = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1763 let Some(fetch) = fetches.get(index) else {
1764 break;
1765 };
1766 let mut report = |bytes: u64| {
1767 (*progress.lock().expect("segment progress lock"))(index, bytes);
1768 };
1769 let result = match fetch {
1770 SegmentFetch::Direct(request) => {
1771 worker.download_segment(request, &mut report)
1772 }
1773 SegmentFetch::Url(url) => worker.fetch_url(url, &mut report),
1774 };
1775 *slots[index].lock().expect("segment result slot") = Some(result);
1776 }
1777 });
1778 }
1779 });
1780 slots
1781 .into_iter()
1782 .map(|slot| slot.into_inner().expect("segment result slot"))
1783 .map(|result| result.expect("every fetch is attempted"))
1784 .collect()
1785 }
1786
1787 fn fetch_url(
1788 &mut self,
1789 url: &str,
1790 on_progress: &mut dyn FnMut(u64),
1791 ) -> Result<Vec<u8>, TransportError> {
1792 let req = self.finish_request(self.agent.get(url), true)?;
1794 let resp = req
1795 .call()
1796 .map_err(|error| segment_error(http_err("segment request failed", error)))?;
1797 read_body(resp, Some(on_progress), self.policy.max_response_bytes)
1798 .map_err(segment_error)
1799 }
1800
1801 fn blob_upload(
1802 &mut self,
1803 blob_id: &str,
1804 bytes: &[u8],
1805 media_type: Option<&str>,
1806 ) -> Result<(), TransportError> {
1807 self.check_request_bytes(bytes.len() as u64)?;
1810 let credentialed = self.credentialed();
1811 let url = format!("{}/blobs/{}", self.base_url, blob_id);
1812 let mut req = self.agent.put(&url).header(
1813 "content-type",
1814 media_type.unwrap_or("application/octet-stream"),
1815 );
1816 for (k, v) in &self.headers {
1817 req = req.header(k.as_str(), v.as_str());
1818 }
1819 let req = self.finish_request(req, credentialed)?;
1820 let resp = req
1821 .send(bytes)
1822 .map_err(|e| http_err("blob upload failed", e))?;
1823 redirect_refused(&resp)?;
1824 Ok(())
1825 }
1826
1827 fn blob_download(&mut self, blob_id: &str) -> Result<BlobDownload, TransportError> {
1828 let credentialed = self.credentialed();
1831 let url = format!("{}/blobs/{}", self.base_url, blob_id);
1832 let mut req = self.agent.get(&url);
1833 for (k, v) in &self.headers {
1834 req = req.header(k.as_str(), v.as_str());
1835 }
1836 let req = self.finish_request(req, credentialed)?;
1837 let resp = req.call().map_err(|cause| {
1838 let status = match &cause {
1839 ureq::Error::StatusCode(status) => Some(*status),
1840 _ => None,
1841 };
1842 let mut error = http_err("blob download failed", cause);
1843 match status {
1844 Some(404) => {
1845 error.code = "blob.not_found".into();
1846 error.message = "blob not found".into();
1847 }
1848 Some(403) => {
1849 error.code = "blob.forbidden".into();
1850 error.message = "blob download forbidden".into();
1851 }
1852 Some(401) => {
1853 error.code = "sync.auth_required".into();
1854 error.message = "host authentication required".into();
1855 }
1856 _ => {}
1857 }
1858 error
1859 })?;
1860 let is_json = resp
1863 .headers()
1864 .get(ureq::http::header::CONTENT_TYPE)
1865 .and_then(|value| value.to_str().ok())
1866 .is_some_and(|value| value.contains("application/json"));
1867 let body = read_body(resp, None, self.policy.max_response_bytes)?;
1868 if is_json {
1869 if let Ok(parsed) = serde_json::from_slice::<serde_json::Value>(&body) {
1870 if let Some(u) = parsed.get("url").and_then(|v| v.as_str()) {
1871 return Ok(BlobDownload::Url {
1872 url: u.to_owned(),
1873 url_expires_at_ms: parsed
1874 .get("urlExpiresAtMs")
1875 .and_then(|v| v.as_i64()),
1876 });
1877 }
1878 }
1879 }
1880 Ok(BlobDownload::Bytes(body))
1881 }
1882
1883 fn fetch_blob_url(&mut self, url: &str) -> Result<Vec<u8>, TransportError> {
1884 self.get_bytes(url, false)
1886 }
1887
1888 fn blob_upload_grant(
1889 &mut self,
1890 blob_id: &str,
1891 byte_length: u64,
1892 media_type: Option<&str>,
1893 ) -> Result<BlobUploadGrant, TransportError> {
1894 let url = format!("{}/blobs/{}/upload-grant", self.base_url, blob_id);
1895 let body = serde_json::json!({
1896 "byteLength": byte_length,
1897 "mediaType": media_type,
1898 })
1899 .to_string();
1900 self.check_request_bytes(body.len() as u64)?;
1901 let credentialed = self.credentialed();
1902 let mut req = self
1903 .agent
1904 .post(&url)
1905 .header("content-type", "application/json");
1906 for (k, v) in &self.headers {
1907 req = req.header(k.as_str(), v.as_str());
1908 }
1909 let req = self.finish_request(req, credentialed)?;
1910 let resp = req
1911 .send(body)
1912 .map_err(|e| http_err("blob upload grant request failed", e))?;
1913 let grant_body = read_body(resp, None, self.policy.max_response_bytes)?;
1914 let parsed: serde_json::Value = serde_json::from_slice(&grant_body)
1915 .map_err(|_| transfer_error("blob upload grant decode failed", "decode", None))?;
1916 if let Some(u) = parsed.get("url").and_then(|v| v.as_str()) {
1917 return Ok(BlobUploadGrant::Url {
1918 url: u.to_owned(),
1919 url_expires_at_ms: parsed.get("urlExpiresAtMs").and_then(|v| v.as_i64()),
1920 });
1921 }
1922 if parsed.get("present").and_then(|v| v.as_bool()) == Some(true) {
1923 return Ok(BlobUploadGrant::Present);
1924 }
1925 Ok(BlobUploadGrant::None)
1926 }
1927
1928 fn blob_put_url(
1929 &mut self,
1930 url: &str,
1931 bytes: &[u8],
1932 media_type: Option<&str>,
1933 ) -> Result<(), TransportError> {
1934 self.check_request_bytes(bytes.len() as u64)?;
1936 let req = self.finish_request(
1937 self.agent.put(url).header(
1938 "content-type",
1939 media_type.unwrap_or("application/octet-stream"),
1940 ),
1941 true,
1942 )?;
1943 let resp = req
1944 .send(bytes)
1945 .map_err(|e| http_err("signed blob upload failed", e))?;
1946 redirect_refused(&resp)?;
1947 Ok(())
1948 }
1949
1950 fn realtime_connect(&mut self) -> Result<(), TransportError> {
1951 if self.outgoing.is_some() {
1952 if self.reader_stop.load(Ordering::SeqCst)
1953 || self
1954 .reader
1955 .as_ref()
1956 .is_some_and(|reader| reader.is_finished())
1957 {
1958 self.shutdown();
1959 } else {
1960 return Ok(());
1961 }
1962 }
1963 use tungstenite::client::IntoClientRequest;
1969 let mut url = url::Url::parse(&self.ws_url)
1970 .map_err(|_| transfer_error("realtime URL invalid", "request", None))?;
1971 if let Some(client_id) = self.realtime_client_id.as_deref() {
1972 let retained_query: Vec<(String, String)> = url
1973 .query_pairs()
1974 .filter(|(key, _)| key != "clientId")
1975 .map(|(key, value)| (key.into_owned(), value.into_owned()))
1976 .collect();
1977 url.set_query(None);
1978 url.query_pairs_mut()
1979 .extend_pairs(retained_query)
1980 .append_pair("clientId", client_id);
1981 }
1982 let mut request = url
1983 .as_str()
1984 .into_client_request()
1985 .map_err(|error| ws_err("realtime request failed", error))?;
1986 {
1987 let out = request.headers_mut();
1988 for (k, v) in &self.headers {
1989 if let (Ok(name), Ok(value)) = (
1990 tungstenite::http::HeaderName::try_from(k.as_str()),
1991 tungstenite::http::HeaderValue::try_from(v.as_str()),
1992 ) {
1993 out.insert(name, value);
1994 }
1995 }
1996 }
1997 if self.policy.redirects == RedirectPolicy::Follow {
2000 return Err(redirect_error(
2001 "realtime establishment does not follow redirects",
2002 None,
2003 ));
2004 }
2005 let mut ws = connect_websocket(request, Instant::now() + CONNECT_TIMEOUT)?;
2006 let stream = match ws.get_mut() {
2007 MaybeTlsStream::Plain(stream) => stream.established(),
2008 MaybeTlsStream::Rustls(stream) => stream.get_mut().established(),
2009 _ => {
2010 return Err(TransportError::new(
2011 "transport.failed",
2012 "unsupported websocket stream",
2013 ))
2014 }
2015 }
2016 .map_err(|e| io_err("realtime clone failed", e))?;
2017 stream
2018 .set_nonblocking(true)
2019 .map_err(|e| io_err("realtime nonblocking failed", e))?;
2020 let poller = Arc::new(Poller::new().map_err(|e| io_err("realtime poller failed", e))?);
2021 unsafe { poller.add(&stream, Event::readable(0)) }
2023 .map_err(|e| io_err("realtime register failed", e))?;
2024 let registration = SocketRegistration {
2025 poller: Arc::clone(&poller),
2026 stream,
2027 };
2028 let (outgoing, queued) = mpsc::sync_channel::<Outgoing>(1);
2029 self.outgoing = Some(Box::new(SocketSender {
2030 queue: outgoing,
2031 poller,
2032 }));
2033 self.reader_stop = Arc::new(AtomicBool::new(false));
2034 self.round = Arc::new(RoundChannel::default());
2035 let inbound = Arc::clone(&self.inbound);
2036 let stop = Arc::clone(&self.reader_stop);
2037 let round = Arc::clone(&self.round);
2038 self.reader = Some(std::thread::spawn(move || {
2039 let mut events = Events::new();
2040 let mut pending: Option<mpsc::Sender<Result<(), TransportError>>> = None;
2041 let failure = 'io: loop {
2042 if stop.load(Ordering::SeqCst) {
2043 let _ = ws.close(None);
2044 let _ = ws.flush();
2045 break TransportError::new(
2046 "sync.transport_failed",
2047 "realtime disconnected mid-round (§8.7)",
2048 );
2049 }
2050 if pending.is_none() {
2051 if let Ok(outgoing) = queued.try_recv() {
2052 pending = Some(outgoing.completed);
2053 if let Err(error) = ws.write(outgoing.message) {
2055 if !is_would_block(&error) {
2056 break ws_err("realtime write failed", error);
2057 }
2058 }
2059 }
2060 }
2061 let mut drained = false;
2063 for _ in 0..64 {
2064 match ws.read() {
2065 Ok(Message::Text(text)) => {
2066 inbound.push(Inbound::Text(text.to_string()))
2067 }
2068 Ok(Message::Binary(bytes)) => {
2069 if let Some(delta) = round.route_binary(&bytes) {
2070 inbound.push(Inbound::Binary(delta));
2071 }
2072 }
2073 Err(error) if is_would_block(&error) => {
2074 drained = true;
2075 break;
2076 }
2077 Ok(Message::Close(_)) => {
2078 let _ = ws.flush();
2079 break 'io TransportError::new(
2080 "sync.transport_failed",
2081 "realtime disconnected mid-round (§8.7)",
2082 );
2083 }
2084 Err(_) => {
2085 break 'io TransportError::new(
2086 "sync.transport_failed",
2087 "realtime disconnected mid-round (§8.7)",
2088 );
2089 }
2090 Ok(_) => {}
2091 }
2092 }
2093 let interest = match ws.flush() {
2094 Ok(()) => {
2095 if let Some(completed) = pending.take() {
2096 let _ = completed.send(Ok(()));
2097 }
2098 Event::readable(0)
2099 }
2100 Err(error) if is_would_block(&error) => Event::all(0),
2101 Err(error) => break ws_err("realtime flush failed", error),
2102 };
2103 if !drained {
2104 continue;
2105 }
2106 if let Err(error) = registration.poller.modify(®istration.stream, interest) {
2107 break io_err("realtime rearm failed", error);
2108 }
2109 events.clear();
2110 if let Err(error) = registration.poller.wait(&mut events, None) {
2111 if error.kind() != std::io::ErrorKind::Interrupted {
2112 break io_err("realtime wait failed", error);
2113 }
2114 }
2115 };
2116 if !stop.load(Ordering::SeqCst) {
2121 inbound.push(Inbound::Disconnected(failure.clone()));
2122 }
2123 if let Some(completed) = pending {
2124 let _ = completed.send(Err(failure.clone()));
2125 }
2126 while let Ok(outgoing) = queued.try_recv() {
2127 let _ = outgoing.completed.send(Err(failure.clone()));
2128 }
2129 round.fail_in_flight(failure);
2130 }));
2131 Ok(())
2132 }
2133
2134 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
2135 self.send_message(Message::Text(text.to_owned().into()))
2136 }
2137
2138 fn realtime_close(&mut self) -> Result<(), TransportError> {
2139 self.shutdown();
2140 Ok(())
2141 }
2142
2143 fn round_deadline(&self) -> Option<Duration> {
2144 NativeTransport::round_deadline(self)
2145 }
2146
2147 fn set_round_deadline(&mut self, deadline: Option<std::time::Instant>) {
2148 NativeTransport::set_round_deadline(self, deadline);
2149 }
2150 }
2151 #[cfg(test)]
2152 mod transfer_tests {
2153 use super::*;
2154 use ureq::unversioned::resolver::DefaultResolver;
2155 use ureq::unversioned::transport::{ConnectionDetails, Connector};
2156
2157 const SECRET: &str = "https://user:password@cdn.example/secret-capability-path?secret-query=value#secret-fragment";
2158
2159 #[derive(Debug)]
2160 struct SecretError;
2161 impl std::fmt::Display for SecretError {
2162 fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2163 panic!("transport error must never be formatted")
2164 }
2165 }
2166 impl std::error::Error for SecretError {}
2167
2168 #[derive(Debug)]
2169 struct FailingConnector(&'static str);
2170 impl Connector for FailingConnector {
2171 type Out = ();
2172 fn connect(
2173 &self,
2174 _: &ConnectionDetails,
2175 _: Option<()>,
2176 ) -> Result<Option<()>, ureq::Error> {
2177 Err(match self.0 {
2178 "io" => ureq::Error::Io(std::io::Error::new(
2179 std::io::ErrorKind::ConnectionRefused,
2180 format!("404 {SECRET}"),
2181 )),
2182 "tls" => ureq::Error::Tls(SECRET),
2183 "request" => ureq::Error::BadUri(SECRET.into()),
2184 "timeout" => ureq::Error::Timeout(ureq::Timeout::Global),
2185 "404" => ureq::Error::StatusCode(404),
2186 "401" => ureq::Error::StatusCode(401),
2187 "403" => ureq::Error::StatusCode(403),
2188 "503" => ureq::Error::StatusCode(503),
2189 _ => ureq::Error::Other(Box::new(SecretError)),
2190 })
2191 }
2192 }
2193
2194 #[test]
2195 fn injected_http_failures_never_expose_capabilities_or_format_the_cause() {
2196 for kind in [
2197 "io", "tls", "request", "timeout", "unknown", "404", "401", "403", "503",
2198 ] {
2199 for operation in [
2200 "sync",
2201 "remote",
2202 "segment",
2203 "signed_segment",
2204 "blob",
2205 "signed_blob",
2206 "upload",
2207 "grant",
2208 "signed_put",
2209 ] {
2210 let mut transport =
2211 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None)
2212 .unwrap();
2213 transport.agent = ureq::Agent::with_parts(
2214 ureq::Agent::config_builder().build(),
2215 FailingConnector(kind),
2216 DefaultResolver::default(),
2217 );
2218 let mut progress = |_| {};
2219 let result = match operation {
2220 "sync" => transport.sync(&[]).map(|_| ()),
2221 "remote" => transport.remote_operation(&[]).map(|_| ()),
2222 "segment" => transport.download_segment(&SegmentRequest { segment_id: "sha256:test".into(), table: "tasks".into(), requested_scopes_json: "{}".into() }, &mut progress).map(|_| ()),
2223 "signed_segment" => transport.fetch_url("http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment", &mut progress).map(|_| ()),
2224 "blob" => transport.blob_download("sha256:test").map(|_| ()),
2225 "signed_blob" => transport.fetch_blob_url("http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment").map(|_| ()),
2226 "upload" => transport.blob_upload("sha256:test", &[], None),
2227 "grant" => transport.blob_upload_grant("sha256:test", 0, None).map(|_| ()),
2228 _ => transport.blob_put_url("http://127.0.0.1/secret-capability-path?secret-query=value#secret-fragment", &[], None),
2229 };
2230 let error = result.unwrap_err();
2231 let expected_code = match (operation, kind) {
2232 (_, "timeout") => "transport.timeout",
2233 ("segment" | "signed_segment", _) => "sync.transport_failed",
2234 ("blob", "404") => "blob.not_found",
2235 ("blob", "401") => "sync.auth_required",
2236 ("blob", "403") => "blob.forbidden",
2237 _ => "transport.failed",
2238 };
2239 assert_eq!(error.code, expected_code, "{operation} {kind}");
2240 let details = error.details.unwrap();
2241 assert_eq!(
2242 details["causeKind"],
2243 match kind {
2244 "io" => "connect",
2245 "404" | "401" | "403" | "503" => "status",
2246 other => other,
2247 }
2248 );
2249 assert_eq!(
2250 details.get("httpStatus").and_then(|value| value.as_u64()),
2251 kind.parse::<u64>().ok()
2252 );
2253 let rendered = format!("{}{}", error.message, details);
2254 for secret in [
2255 "http://",
2256 "https://",
2257 "password",
2258 "secret-capability-path",
2259 "secret-query",
2260 "secret-fragment",
2261 ] {
2262 assert!(!rendered.contains(secret), "{operation} {kind}");
2263 }
2264 assert!(details.get("causeMessage").is_none());
2265 assert!(details.get("path").is_none());
2266 }
2267 }
2268 }
2269
2270 #[test]
2271 fn injected_body_and_websocket_errors_do_not_expose_payloads_or_source_chains() {
2272 struct FailingRead;
2273 impl std::io::Read for FailingRead {
2274 fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
2275 Err(std::io::Error::other(SECRET))
2276 }
2277 }
2278 let response = ureq::http::Response::builder()
2279 .status(200)
2280 .body(ureq::Body::builder().reader(FailingRead))
2281 .unwrap();
2282 let error = segment_error(read_body(response, None, None).unwrap_err());
2283 assert_eq!(error.code, "sync.transport_failed");
2284 assert_eq!(error.message, "segment transfer failed");
2285 assert_eq!(
2286 error.details,
2287 Some(serde_json::json!({ "causeKind": "body", "httpStatus": 200 }))
2288 );
2289 for cause in [
2290 tungstenite::Error::Io(std::io::Error::other(SecretError)),
2291 tungstenite::Error::Http(Box::new(
2292 tungstenite::http::Response::builder()
2293 .status(403)
2294 .header("location", SECRET)
2295 .body(Some(SECRET.as_bytes().to_vec()))
2296 .unwrap(),
2297 )),
2298 ] {
2299 let error = ws_err("realtime connection failed", cause);
2300 assert_eq!(error.message, "realtime connection failed");
2301 let details = error.details.unwrap();
2302 assert!(details.get("path").is_none());
2303 assert!(details.get("causeMessage").is_none());
2304 assert!(!details.to_string().contains("secret"));
2305 }
2306 }
2307
2308 #[test]
2309 fn body_read_timeouts_are_classified_through_ureq_wrapping() {
2310 let direct = ureq::Error::Timeout(ureq::Timeout::Global).into_io();
2312 assert!(is_timeout_io(&direct));
2313 let wrapped = ureq::Error::Decompress(
2315 "gzip",
2316 ureq::Error::Timeout(ureq::Timeout::Global).into_io(),
2317 )
2318 .into_io();
2319 assert!(is_timeout_io(&wrapped));
2320 assert!(is_timeout_io(&std::io::Error::new(
2322 std::io::ErrorKind::TimedOut,
2323 "plain"
2324 )));
2325 assert!(!is_timeout_io(&std::io::Error::other("other")));
2326 assert!(!is_timeout_io(&ureq::Error::StatusCode(500).into_io()));
2327 }
2328
2329 #[test]
2330 fn body_read_wrapped_timeouts_surface_the_typed_code() {
2331 struct TimeoutRead(bool);
2332 impl std::io::Read for TimeoutRead {
2333 fn read(&mut self, _: &mut [u8]) -> std::io::Result<usize> {
2334 Err(if self.0 {
2335 ureq::Error::Timeout(ureq::Timeout::Global).into_io()
2336 } else {
2337 ureq::Error::Decompress(
2338 "gzip",
2339 ureq::Error::Timeout(ureq::Timeout::Global).into_io(),
2340 )
2341 .into_io()
2342 })
2343 }
2344 }
2345 for compressed in [false, true] {
2346 let response = ureq::http::Response::builder()
2347 .status(200)
2348 .body(ureq::Body::builder().reader(TimeoutRead(compressed)))
2349 .unwrap();
2350 let error = read_body(response, None, None).unwrap_err();
2351 assert_eq!(error.code, "transport.timeout", "{compressed}");
2352 assert_eq!(
2353 error.details,
2354 Some(serde_json::json!({ "causeKind": "timeout" }))
2355 );
2356 }
2357 }
2358
2359 #[test]
2360 fn policy_validation_rejects_zero_and_unrepresentable_bounds() {
2361 assert_eq!(
2362 HostTransportPolicy {
2363 request_timeout: Some(Duration::ZERO),
2364 ..Default::default()
2365 }
2366 .validate()
2367 .unwrap_err(),
2368 "sync.invalid_request: transport config requestTimeoutMs is out of range"
2369 );
2370 assert_eq!(
2371 HostTransportPolicy {
2372 round_deadline: Some(Duration::MAX),
2373 ..Default::default()
2374 }
2375 .validate()
2376 .unwrap_err(),
2377 "sync.invalid_request: transport config roundDeadlineMs is out of range"
2378 );
2379 assert!(HostTransportPolicy {
2380 max_request_bytes: Some(0),
2381 ..Default::default()
2382 }
2383 .validate()
2384 .is_err());
2385 assert!(HostTransportPolicy {
2386 max_response_bytes: Some(0),
2387 ..Default::default()
2388 }
2389 .validate()
2390 .is_err());
2391 assert!(HostTransportPolicy::default().validate().is_ok());
2394 assert_eq!(
2395 HostTransportPolicy::from_config(&serde_json::json!({ "requestTimeoutMs": 0 }))
2396 .unwrap_err(),
2397 "sync.invalid_request: transport config requestTimeoutMs must be a positive integer"
2398 );
2399 }
2400
2401 #[test]
2402 fn set_policy_rejects_invalid_and_preserves_the_current_policy() {
2403 let mut transport =
2404 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None).unwrap();
2405 assert_eq!(transport.policy.max_response_bytes, None);
2406 let error = transport
2407 .set_policy(HostTransportPolicy {
2408 max_response_bytes: Some(0),
2409 ..Default::default()
2410 })
2411 .unwrap_err();
2412 assert_eq!(
2413 error,
2414 "sync.invalid_request: transport config maxResponseBytes must be a positive integer"
2415 );
2416 assert_eq!(transport.policy.max_response_bytes, None);
2417 }
2418
2419 #[test]
2420 fn an_expired_round_deadline_is_typed_and_cleared_for_the_next_round() {
2421 let mut transport =
2422 NativeTransport::new("http://127.0.0.1:9", &serde_json::json!({}), None).unwrap();
2423 transport.set_round_deadline(Some(std::time::Instant::now() - Duration::from_secs(1)));
2424 let error = transport.blob_download("sha256:test").unwrap_err();
2425 assert_eq!(error.code, "transport.timeout");
2426 assert_eq!(
2427 error.details,
2428 Some(serde_json::json!({ "causeKind": "round_deadline" }))
2429 );
2430 transport.set_round_deadline(None);
2432 assert!(transport.round_remaining().is_ok());
2433 }
2434
2435 #[test]
2436 fn an_expired_round_deadline_refuses_before_queuing_a_realtime_send() {
2437 let mut transport =
2438 NativeTransport::new("http://127.0.0.1:9", &serde_json::json!({}), None).unwrap();
2439 transport.set_round_deadline(Some(std::time::Instant::now() - Duration::from_secs(1)));
2440 let error = transport.realtime_send("blocked").unwrap_err();
2444 assert_eq!(error.code, "transport.timeout");
2445 assert_eq!(
2446 error.details,
2447 Some(serde_json::json!({ "causeKind": "round_deadline" }))
2448 );
2449 transport.set_round_deadline(None);
2452 let error = transport.realtime_send("blocked").unwrap_err();
2453 assert_eq!(error.code, "transport.failed");
2454 }
2455
2456 #[test]
2457 fn the_request_timeout_bounds_phases_and_the_round_deadline_bounds_the_call() {
2458 #[derive(Debug)]
2459 struct TimeoutRecording(Arc<Mutex<Vec<Option<Duration>>>>);
2460 impl Connector for TimeoutRecording {
2461 type Out = ();
2462 fn connect(
2463 &self,
2464 details: &ConnectionDetails,
2465 _: Option<()>,
2466 ) -> Result<Option<()>, ureq::Error> {
2467 self.0
2468 .lock()
2469 .unwrap()
2470 .push(details.config.timeouts().global);
2471 Err(ureq::Error::ConnectionFailed)
2472 }
2473 }
2474 let mut transport =
2475 NativeTransport::new("http://127.0.0.1", &serde_json::json!({}), None).unwrap();
2476 transport
2477 .set_policy(HostTransportPolicy {
2478 request_timeout: Some(Duration::from_millis(1234)),
2479 ..Default::default()
2480 })
2481 .unwrap();
2482 let timeouts = transport.agent.config().timeouts();
2483 for phase in [
2484 timeouts.resolve,
2485 timeouts.connect,
2486 timeouts.send_request,
2487 timeouts.recv_response,
2488 ] {
2489 assert_eq!(phase, Some(Duration::from_millis(1234)));
2490 }
2491 assert_eq!(timeouts.global, None, "the request bound is not end to end");
2492 assert_eq!(timeouts.recv_body, None, "a body is bounded by silence");
2493 let seen = Arc::new(Mutex::new(Vec::new()));
2494 transport.agent = ureq::Agent::with_parts(
2495 ureq::Agent::config_builder().build(),
2496 TimeoutRecording(Arc::clone(&seen)),
2497 DefaultResolver::default(),
2498 );
2499 let _ = transport.blob_upload("sha256:test", &[0u8; 4], None);
2500 transport.set_round_deadline(Some(std::time::Instant::now() + Duration::from_secs(60)));
2501 let _ = transport.blob_upload("sha256:test", &[0u8; 4], None);
2502 let seen = seen.lock().unwrap();
2503 assert_eq!(seen[0], None, "no round deadline, no call ceiling");
2504 assert!(seen[1].is_some_and(|ceiling| ceiling <= Duration::from_secs(60)));
2505 }
2506
2507 #[test]
2508 fn the_idle_bound_replaces_an_unbounded_or_longer_wait_and_keeps_a_shorter_one() {
2509 let idle = Duration::from_secs(30);
2510 let unbounded = NextTimeout {
2511 after: UreqDuration::NotHappening,
2512 reason: UreqTimeout::Global,
2513 };
2514 assert_eq!(idle_bound(unbounded, idle).after, UreqDuration::Exact(idle));
2515 let longer = NextTimeout {
2516 after: UreqDuration::from_secs(90),
2517 reason: UreqTimeout::Global,
2518 };
2519 assert_eq!(idle_bound(longer, idle).after, UreqDuration::Exact(idle));
2520 let shorter = NextTimeout {
2521 after: UreqDuration::from_secs(5),
2522 reason: UreqTimeout::Global,
2523 };
2524 assert_eq!(idle_bound(shorter, idle), shorter);
2525 }
2526
2527 fn shaped_link(
2532 total: u64,
2533 chunk: u64,
2534 latency: Duration,
2535 rate: u64,
2536 stalls: &[(Duration, Duration)],
2537 ) -> Vec<Duration> {
2538 let mut at = latency;
2539 let mut arrivals = Vec::new();
2540 let mut left = total;
2541 while left > 0 {
2542 let piece = left.min(chunk);
2543 at += Duration::from_micros(piece * 1_000_000 / rate);
2544 for (start, length) in stalls {
2545 if at > *start && at < *start + *length {
2546 at = *start + *length;
2547 }
2548 }
2549 arrivals.push(at);
2550 left -= piece;
2551 }
2552 arrivals
2553 }
2554
2555 fn round_over(
2559 arrivals: &[Duration],
2560 idle: Duration,
2561 ceiling: Option<Duration>,
2562 ) -> Result<Duration, (Duration, bool)> {
2563 let base = Instant::now();
2564 let mut last = base;
2565 for arrival in arrivals {
2566 let (deadline, ceiling_binds) =
2567 round_expiry(last, idle, ceiling.map(|ceiling| base + ceiling));
2568 if base + *arrival > deadline {
2569 return Err((deadline - base, ceiling_binds));
2570 }
2571 last = base + *arrival;
2572 }
2573 Ok(last - base)
2574 }
2575
2576 #[test]
2577 fn a_slow_link_that_keeps_delivering_never_trips_the_idle_bound() {
2578 let link = shaped_link(
2580 8_600_000,
2581 64 * 1024,
2582 Duration::from_millis(80),
2583 100_000,
2584 &[],
2585 );
2586 let finished = round_over(&link, ROUND_IDLE_TIMEOUT, None).unwrap();
2587 assert!(
2588 finished > ROUND_IDLE_TIMEOUT * 2,
2589 "the transfer outlasts the old whole-transfer bound ({finished:?})"
2590 );
2591 }
2592
2593 #[test]
2594 fn a_stall_shorter_than_the_idle_bound_is_survived_and_a_longer_one_is_not() {
2595 let link = |length| {
2596 shaped_link(
2597 4_000_000,
2598 64 * 1024,
2599 Duration::from_millis(80),
2600 100_000,
2601 &[(Duration::from_secs(10), length)],
2602 )
2603 };
2604 round_over(&link(Duration::from_secs(29)), ROUND_IDLE_TIMEOUT, None).unwrap();
2605 let (expired, ceiling_bound) =
2606 round_over(&link(Duration::from_secs(31)), ROUND_IDLE_TIMEOUT, None).unwrap_err();
2607 assert!(!ceiling_bound);
2608 assert!(
2609 expired >= Duration::from_secs(30) && expired < Duration::from_secs(41),
2610 "the bound counts from the last chunk before the stall ({expired:?})"
2611 );
2612 }
2613
2614 #[test]
2615 fn the_host_ceiling_still_bounds_a_link_that_keeps_delivering() {
2616 let link = shaped_link(
2617 8_600_000,
2618 64 * 1024,
2619 Duration::from_millis(80),
2620 100_000,
2621 &[],
2622 );
2623 let (expired, ceiling_bound) =
2624 round_over(&link, ROUND_IDLE_TIMEOUT, Some(Duration::from_secs(60))).unwrap_err();
2625 assert!(ceiling_bound);
2626 assert_eq!(expired, Duration::from_secs(60));
2627 }
2628
2629 #[test]
2630 fn an_expired_round_deadline_bounds_the_socket_wait() {
2631 let channel = RoundChannel::default();
2632 let error = channel
2633 .wait(
2634 Some(std::time::Instant::now() - Duration::from_secs(1)),
2635 ROUND_IDLE_TIMEOUT,
2636 )
2637 .unwrap_err();
2638 assert_eq!(error.code, "transport.timeout");
2639 assert_eq!(
2640 error.details,
2641 Some(serde_json::json!({ "causeKind": "round_deadline" }))
2642 );
2643 }
2644 }
2645}
2646
2647#[cfg(all(test, feature = "native-transport"))]
2648mod tests {
2649 use super::*;
2650 use std::io::{BufRead, BufReader, Write};
2651
2652 #[test]
2653 fn detached_round_uses_the_owned_socket_and_shutdown_releases_it() {
2654 use ssp2::model::{Frame, Message, MsgKind};
2655 use std::sync::mpsc::channel;
2656 for cancelled in [false, true] {
2657 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2658 let base = format!("http://{}", listener.local_addr().unwrap());
2659 let (entered_tx, entered_rx) = channel();
2660 let (release_tx, release_rx) = channel();
2661 let server = std::thread::spawn(move || {
2662 let (socket, _) = listener.accept().unwrap();
2663 let mut socket = tungstenite::accept(socket).unwrap();
2664 let request = socket.read().unwrap().into_data();
2665 assert_eq!(request[0], crate::REALTIME_TAG_ROUND);
2666 assert_eq!(
2667 ssp2::decode_message(&request[1..]).unwrap().msg_kind,
2668 MsgKind::Request
2669 );
2670 entered_tx.send(()).unwrap();
2671 release_rx.recv().unwrap();
2672 if !cancelled {
2673 let response = ssp2::encode_message(&Message {
2674 wire_version: ssp2::decode::WIRE_VERSION,
2675 msg_kind: MsgKind::Response,
2676 frames: vec![Frame::RespHeader {
2677 required_schema_version: None,
2678 latest_schema_version: None,
2679 log_epoch: Some("epoch".into()),
2680 reset_required: Some(true),
2681 }],
2682 });
2683 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
2684 chunk.extend(response);
2685 socket
2686 .send(tungstenite::Message::Binary(chunk.into()))
2687 .unwrap();
2688 }
2689 });
2690 let mut owner =
2691 HostTransport::new_from_config(&serde_json::json!({"baseUrl":base})).unwrap();
2692 owner.realtime_connect_for_client("detached-round").unwrap();
2693 let mut detached = owner.fork_round();
2694 let network = std::thread::spawn(move || {
2695 let request = ssp2::encode_message(&Message {
2696 wire_version: ssp2::decode::WIRE_VERSION,
2697 msg_kind: MsgKind::Request,
2698 frames: vec![
2699 Frame::ReqHeader {
2700 client_id: "detached-round".into(),
2701 schema_version: 1,
2702 log_epoch: None,
2703 },
2704 Frame::PullHeader {
2705 limit_commits: 0,
2706 limit_snapshot_rows: 0,
2707 max_snapshot_pages: 0,
2708 accept: 3,
2709 },
2710 ],
2711 });
2712 detached.realtime_sync(&request)
2713 });
2714 entered_rx.recv().unwrap();
2715 owner.set_headers(vec![("authorization".into(), "Bearer rotated".into())]);
2716 assert!(owner.take_inbound().is_empty());
2717 if cancelled {
2718 owner.shutdown();
2719 }
2720 release_tx.send(()).unwrap();
2721 let response = network.join().unwrap();
2722 if cancelled {
2723 assert!(response.is_err());
2724 } else {
2725 assert_eq!(
2726 ssp2::decode_message(&response.unwrap()).unwrap().msg_kind,
2727 MsgKind::Response
2728 );
2729 owner.shutdown();
2730 }
2731 server.join().unwrap();
2732 }
2733 }
2734
2735 fn minimal_response() -> Vec<u8> {
2736 use ssp2::model::{Frame, Message, MsgKind};
2737 ssp2::encode_message(&Message {
2738 wire_version: ssp2::decode::WIRE_VERSION,
2739 msg_kind: MsgKind::Response,
2740 frames: vec![Frame::RespHeader {
2741 required_schema_version: None,
2742 latest_schema_version: None,
2743 log_epoch: Some("epoch".into()),
2744 reset_required: Some(true),
2745 }],
2746 })
2747 }
2748
2749 #[test]
2750 fn a_stalled_response_body_times_out_on_silence() {
2751 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2752 let base = format!("http://{}", listener.local_addr().unwrap());
2753 let server = std::thread::spawn(move || {
2754 let (mut socket, _) = listener.accept().unwrap();
2755 let mut reader = BufReader::new(socket.try_clone().unwrap());
2756 loop {
2757 let mut line = String::new();
2758 reader.read_line(&mut line).unwrap();
2759 if line == "\r\n" {
2760 break;
2761 }
2762 }
2763 socket
2764 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n0123456789")
2765 .unwrap();
2766 let mut rest = Vec::new();
2768 let _ = std::io::Read::read_to_end(&mut reader, &mut rest);
2769 });
2770 let mut transport = HostTransport::new_from_config(
2771 &serde_json::json!({"baseUrl": base, "requestTimeoutMs": 200}),
2772 )
2773 .unwrap();
2774 let mut received = 0;
2775 let error = transport
2776 .download_segment(
2777 &SegmentRequest {
2778 segment_id: "sha256:test".into(),
2779 table: "tasks".into(),
2780 requested_scopes_json: "{}".into(),
2781 },
2782 &mut |bytes| received = bytes,
2783 )
2784 .unwrap_err();
2785 assert_eq!(error.code, "transport.timeout");
2786 assert_eq!(
2787 error.details,
2788 Some(serde_json::json!({ "causeKind": "timeout" }))
2789 );
2790 server.join().unwrap();
2791 }
2792
2793 #[test]
2794 fn a_timed_out_socket_round_closes_the_socket_and_reports_the_loss() {
2795 use std::sync::mpsc::channel;
2796 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2797 let base = format!("http://{}", listener.local_addr().unwrap());
2798 let (closed_tx, closed_rx) = channel();
2799 let server = std::thread::spawn(move || {
2800 let (socket, _) = listener.accept().unwrap();
2801 let mut socket = tungstenite::accept(socket).unwrap();
2802 socket.read().unwrap();
2803 let response = minimal_response();
2805 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
2806 chunk.extend_from_slice(&response[..response.len() / 2]);
2807 socket
2808 .send(tungstenite::Message::Binary(chunk.into()))
2809 .unwrap();
2810 while let Ok(message) = socket.read() {
2812 if matches!(message, tungstenite::Message::Close(_)) {
2813 break;
2814 }
2815 }
2816 closed_tx.send(()).unwrap();
2817 });
2818 let mut transport =
2819 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
2820 transport.realtime_connect_for_client("slow").unwrap();
2821 transport.set_round_deadline(Some(
2822 std::time::Instant::now() + std::time::Duration::from_millis(300),
2823 ));
2824 let error = transport.realtime_sync(b"request").unwrap_err();
2825 assert_eq!(error.code, "transport.timeout");
2826 closed_rx.recv().unwrap();
2827 let lost = transport
2828 .take_inbound()
2829 .into_iter()
2830 .filter(|inbound| matches!(inbound, Inbound::Disconnected(e) if e.code == "transport.timeout"))
2831 .count();
2832 assert_eq!(lost, 1, "the loss is reported once");
2833 server.join().unwrap();
2834 }
2835
2836 #[test]
2837 fn a_request_larger_than_a_chunk_rides_as_several_round_chunks() {
2838 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2839 let base = format!("http://{}", listener.local_addr().unwrap());
2840 let request: Vec<u8> = (0..3 * 64 * 1024 + 5).map(|i| (i % 251) as u8).collect();
2841 let expected = request.clone();
2842 let server = std::thread::spawn(move || {
2843 let (socket, _) = listener.accept().unwrap();
2844 let mut socket = tungstenite::accept(socket).unwrap();
2845 let mut body = Vec::new();
2846 let mut messages = 0;
2847 while body.len() < expected.len() {
2848 let message = socket.read().unwrap().into_data();
2849 assert_eq!(message[0], crate::REALTIME_TAG_ROUND);
2850 body.extend_from_slice(&message[1..]);
2851 messages += 1;
2852 }
2853 assert_eq!(body, expected);
2854 assert_eq!(messages, 4);
2855 let mut chunk = vec![crate::REALTIME_TAG_ROUND];
2856 chunk.extend(minimal_response());
2857 socket
2858 .send(tungstenite::Message::Binary(chunk.into()))
2859 .unwrap();
2860 });
2861 let mut transport =
2862 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
2863 transport.realtime_connect_for_client("chunked").unwrap();
2864 let response = transport.realtime_sync(&request).unwrap();
2865 assert_eq!(response, minimal_response());
2866 transport.shutdown();
2867 server.join().unwrap();
2868 }
2869
2870 #[test]
2871 fn segment_http_failures_have_named_codes_and_safe_cause_details() {
2872 for signed in [false, true] {
2873 for body_failure in [false, true] {
2874 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2875 let base = format!("http://{}", listener.local_addr().unwrap());
2876 let server = std::thread::spawn(move || {
2877 let (mut socket, _) = listener.accept().unwrap();
2878 let mut reader = BufReader::new(socket.try_clone().unwrap());
2879 loop {
2880 let mut line = String::new();
2881 reader.read_line(&mut line).unwrap();
2882 if line == "\r\n" {
2883 break;
2884 }
2885 assert!(!line.is_empty());
2886 }
2887 socket.write_all(if body_failure {
2888 b"HTTP/1.1 200 OK\r\nContent-Length: 999\r\nConnection: close\r\n\r\nshort"
2889 } else {
2890 b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
2891 }).unwrap();
2892 });
2893 let mut transport =
2894 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
2895 let mut progress = |_| {};
2896 let error = if signed {
2897 transport.fetch_url(&format!("{base}/signed?signature=secret"), &mut progress)
2898 } else {
2899 transport.download_segment(
2900 &SegmentRequest {
2901 segment_id: "sha256:test".into(),
2902 table: "tasks".into(),
2903 requested_scopes_json: "{}".into(),
2904 },
2905 &mut progress,
2906 )
2907 }
2908 .unwrap_err();
2909 assert_eq!(error.code, "sync.transport_failed");
2910 assert_eq!(error.message, "segment transfer failed");
2911 let details = error.details.unwrap();
2912 assert!(details.get("path").is_none());
2913 assert!(details.get("causeMessage").is_none());
2914 assert_eq!(
2915 details["causeKind"],
2916 if body_failure { "body" } else { "status" }
2917 );
2918 assert_eq!(details["httpStatus"], if body_failure { 200 } else { 403 });
2919 server.join().unwrap();
2920 }
2921 }
2922 }
2923
2924 #[test]
2925 fn blob_status_classification_uses_the_actual_http_response() {
2926 for (status, expected) in [
2927 (404, "blob.not_found"),
2928 (401, "sync.auth_required"),
2929 (403, "blob.forbidden"),
2930 (500, "transport.failed"),
2931 ] {
2932 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2933 let base = format!("http://{}", listener.local_addr().unwrap());
2934 let server = std::thread::spawn(move || {
2935 let (mut socket, _) = listener.accept().unwrap();
2936 let mut reader = BufReader::new(socket.try_clone().unwrap());
2937 loop {
2938 let mut line = String::new();
2939 reader.read_line(&mut line).unwrap();
2940 if line == "\r\n" {
2941 break;
2942 }
2943 assert!(!line.is_empty());
2944 }
2945 let body = r#"{"code":"blob.not_found","message":"https://host/secret-path?secret-query#secret-fragment","retryable":false}"#;
2946 write!(socket, "HTTP/1.1 {status} Error\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();
2947 });
2948 let mut transport =
2949 HostTransport::new_from_config(&serde_json::json!({"baseUrl":base})).unwrap();
2950 let error = transport.blob_download("sha256:test").unwrap_err();
2951 assert_eq!(error.code, expected);
2952 assert_eq!(
2953 error.details,
2954 Some(serde_json::json!({ "causeKind":"status", "httpStatus":status }))
2955 );
2956 assert!(!error.message.contains("secret"));
2957 assert!(!error.message.contains("http"));
2958 server.join().unwrap();
2959 }
2960 }
2961
2962 #[test]
2963 fn refused_segment_connection_has_a_named_code_without_an_invented_status() {
2964 for signed in [false, true] {
2965 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2966 let base = format!("http://{}", listener.local_addr().unwrap());
2967 drop(listener);
2968 let mut transport =
2969 HostTransport::new_from_config(&serde_json::json!({"baseUrl": base})).unwrap();
2970 let mut progress = |_| {};
2971 let error = if signed {
2972 transport.fetch_url(&format!("{base}/signed?signature=secret"), &mut progress)
2973 } else {
2974 transport.download_segment(
2975 &SegmentRequest {
2976 segment_id: "sha256:test".into(),
2977 table: "tasks".into(),
2978 requested_scopes_json: "{}".into(),
2979 },
2980 &mut progress,
2981 )
2982 }
2983 .unwrap_err();
2984 assert_eq!(error.code, "sync.transport_failed");
2985 assert!(error.details.as_ref().unwrap().get("httpStatus").is_none());
2986 }
2987 }
2988
2989 #[test]
2990 fn segment_http_reports_intermediate_bytes_for_direct_and_signed_urls() {
2991 for signed in [false, true] {
2992 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2993 let base = format!("http://{}", listener.local_addr().unwrap());
2994 let server = std::thread::spawn(move || {
2995 let (mut socket, _) = listener.accept().unwrap();
2996 let mut reader = BufReader::new(socket.try_clone().unwrap());
2997 let mut headers = String::new();
2998 loop {
2999 let mut line = String::new();
3000 reader.read_line(&mut line).unwrap();
3001 if line == "\r\n" {
3002 break;
3003 }
3004 assert!(!line.is_empty());
3005 headers.push_str(&line.to_ascii_lowercase());
3006 }
3007 assert_eq!(headers.contains("authorization: bearer test"), !signed);
3008 assert_eq!(headers.contains("x-syncular-scopes:"), !signed);
3009 let bytes = vec![7; 128 * 1024 + 17];
3010 write!(
3011 socket,
3012 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
3013 bytes.len()
3014 )
3015 .unwrap();
3016 socket.write_all(&bytes).unwrap();
3017 });
3018 let mut transport = HostTransport::from_config(
3019 &serde_json::json!({"baseUrl": base, "headers": {"authorization": "Bearer test"}}),
3020 )
3021 .unwrap();
3022 let mut updates = Vec::new();
3023 let mut progress = |bytes| updates.push(bytes);
3024 let bytes = if signed {
3025 transport
3026 .fetch_url(&format!("{base}/signed"), &mut progress)
3027 .unwrap()
3028 } else {
3029 transport
3030 .download_segment(
3031 &SegmentRequest {
3032 segment_id: "sha256:test".into(),
3033 table: "tasks".into(),
3034 requested_scopes_json: "{}".into(),
3035 },
3036 &mut progress,
3037 )
3038 .unwrap()
3039 };
3040 assert_eq!(bytes.len(), 128 * 1024 + 17);
3041 assert!(updates.iter().any(|n| *n > 0 && *n < bytes.len() as u64));
3042 assert_eq!(updates.last(), Some(&(bytes.len() as u64)));
3043 assert!(updates.windows(2).all(|pair| pair[0] < pair[1]));
3044 server.join().unwrap();
3045 }
3046 }
3047
3048 #[test]
3049 fn request_over_cap_is_refused_before_network() {
3050 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3051 listener.set_nonblocking(true).unwrap();
3052 let base = format!("http://{}", listener.local_addr().unwrap());
3053 let mut transport =
3054 HostTransport::from_config(&serde_json::json!({"baseUrl": base, "maxRequestBytes": 4}))
3055 .unwrap();
3056 let error = transport
3057 .blob_upload("sha256:test", &[0u8; 8], None)
3058 .unwrap_err();
3059 assert_eq!(error.code, "transport.request_too_large");
3060 assert_eq!(
3061 error.details,
3062 Some(serde_json::json!({ "limit": 4, "size": 8 }))
3063 );
3064 assert!(
3065 matches!(listener.accept(), Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock),
3066 "the request cap is checked before any connection"
3067 );
3068 }
3069
3070 #[test]
3071 fn credential_bearing_redirects_are_refused_without_contacting_the_destination() {
3072 for credentials in ["headers", "userinfo", "password"] {
3073 for redirects in ["deny", "follow"] {
3074 let destination = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3075 destination.set_nonblocking(true).unwrap();
3076 let location = format!("http://{}/landed", destination.local_addr().unwrap());
3077 let origin = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3078 let host = origin.local_addr().unwrap();
3079 let (base, headers) = match credentials {
3080 "headers" => (
3081 format!("http://{host}"),
3082 serde_json::json!({"authorization": "Bearer secret"}),
3083 ),
3084 "userinfo" => (format!("http://user:secret@{host}"), serde_json::json!({})),
3085 _ => (format!("http://:secret@{host}"), serde_json::json!({})),
3086 };
3087 let server = std::thread::spawn(move || {
3088 let (mut socket, _) = origin.accept().unwrap();
3089 let mut reader = BufReader::new(socket.try_clone().unwrap());
3090 loop {
3091 let mut line = String::new();
3092 reader.read_line(&mut line).unwrap();
3093 if line == "\r\n" {
3094 break;
3095 }
3096 }
3097 let _ = write!(
3098 socket,
3099 "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
3100 );
3101 });
3102 let mut transport = HostTransport::from_config(&serde_json::json!({
3103 "baseUrl": base,
3104 "redirects": redirects,
3105 "headers": headers,
3106 }))
3107 .unwrap();
3108 let error = transport.blob_download("sha256:test").unwrap_err();
3109 assert_eq!(
3110 error.code, "transport.redirect",
3111 "{credentials} {redirects}"
3112 );
3113 let details = error.details.unwrap();
3114 assert_eq!(details["causeKind"], "redirect");
3115 assert_eq!(details["httpStatus"], 302);
3116 assert!(
3117 matches!(destination.accept(), Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock),
3118 "{credentials} {redirects}: the destination is never contacted"
3119 );
3120 server.join().unwrap();
3121 }
3122 }
3123 }
3124
3125 #[test]
3126 fn websocket_handshake_redirects_obey_the_redirect_policy() {
3127 for credential in ["headers", "userinfo", "query", "none"] {
3134 let redirects = "deny";
3135 let destination = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3136 let destination_addr = destination.local_addr().unwrap();
3137 let location = format!("ws://{destination_addr}/realtime");
3138 let landed = Arc::new(Mutex::new(Vec::new()));
3142 let landed_writer = Arc::clone(&landed);
3143 let destination_thread = std::thread::spawn(move || loop {
3144 let (mut socket, _) = destination.accept().unwrap();
3145 let mut reader = BufReader::new(socket.try_clone().unwrap());
3146 let mut head = String::new();
3147 loop {
3148 let mut line = String::new();
3149 reader.read_line(&mut line).unwrap();
3150 if line == "\r\n" {
3151 break;
3152 }
3153 head.push_str(&line);
3154 }
3155 let sentinel = head.starts_with("GET /sentinel");
3156 landed_writer.lock().unwrap().push(head);
3157 let _ = write!(
3158 socket,
3159 "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
3160 );
3161 if sentinel {
3162 break;
3163 }
3164 });
3165 let origin = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3166 let origin_addr = origin.local_addr().unwrap();
3167 let (headers, ws_url) = match credential {
3168 "headers" => (serde_json::json!({"authorization": "Bearer secret"}), None),
3169 "userinfo" => (
3170 serde_json::json!({}),
3171 Some(format!("ws://user:secret@{origin_addr}/realtime")),
3172 ),
3173 "query" => (
3174 serde_json::json!({}),
3175 Some(format!("ws://{origin_addr}/realtime?token=secret")),
3176 ),
3177 _ => (serde_json::json!({}), None),
3178 };
3179 let (head_tx, head_rx) = std::sync::mpsc::channel();
3180 let origin_thread = std::thread::spawn(move || {
3181 let (mut socket, _) = origin.accept().unwrap();
3182 let mut reader = BufReader::new(socket.try_clone().unwrap());
3183 let mut head = String::new();
3184 loop {
3185 let mut line = String::new();
3186 reader.read_line(&mut line).unwrap();
3187 if line == "\r\n" {
3188 break;
3189 }
3190 head.push_str(&line);
3191 }
3192 let _ = head_tx.send(head);
3193 let _ = write!(
3194 socket,
3195 "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
3196 );
3197 });
3198 let mut config = serde_json::json!({
3199 "baseUrl": format!("http://{origin_addr}"),
3200 "headers": headers,
3201 "redirects": redirects,
3202 });
3203 if let Some(ws_url) = &ws_url {
3204 config["wsUrl"] = serde_json::json!(ws_url);
3205 }
3206 let mut transport = HostTransport::from_config(&config).unwrap();
3207 let result = transport.realtime_connect();
3208 let mut sentinel = std::net::TcpStream::connect(destination_addr).unwrap();
3211 let _ = write!(sentinel, "GET /sentinel HTTP/1.1\r\nHost: sentinel\r\n\r\n");
3212 destination_thread.join().unwrap();
3213 let landed = landed.lock().unwrap().clone();
3214 origin_thread.join().unwrap();
3215 let error = result.unwrap_err();
3216 assert_eq!(
3217 error.code, "transport.redirect",
3218 "{credential}: a refused handshake redirect surfaces the redirect code"
3219 );
3220 let details = error.details.unwrap();
3221 assert_eq!(details["causeKind"], "redirect");
3222 assert_eq!(details["httpStatus"], 302);
3223 assert_eq!(
3224 landed.len(),
3225 1,
3226 "{credential}: the redirect target is never contacted: {landed:?}"
3227 );
3228 assert!(landed[0].starts_with("GET /sentinel"), "{landed:?}");
3229 let head = head_rx.recv().unwrap();
3230 if credential == "userinfo" {
3231 assert!(
3232 !head.contains("secret"),
3233 "the handshake request line drops URL userinfo"
3234 );
3235 }
3236 if credential == "query" {
3237 assert!(
3238 head.contains("token=secret"),
3239 "the handshake request line carries the realtime query"
3240 );
3241 }
3242 }
3243 }
3244
3245 const GZIP_BOMB: [u8; 548] = [
3248 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xed, 0xc9, 0xb1, 0x11, 0x80,
3249 0x20, 0x10, 0x00, 0xb0, 0x59, 0x1c, 0x80, 0xa1, 0xfe, 0x90, 0xb3, 0x41, 0xf0, 0x7c, 0x2d,
3250 0xd8, 0xde, 0x09, 0xdc, 0x20, 0x69, 0x93, 0x6b, 0xd4, 0xb7, 0xc7, 0x5d, 0xb2, 0x1d, 0x67,
3251 0x1b, 0x4f, 0xb9, 0x62, 0xf5, 0x19, 0xfb, 0x96, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3252 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3253 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3254 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3255 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3256 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3257 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3258 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3259 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3260 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3261 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3262 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3263 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3264 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3265 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3266 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3267 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3268 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3269 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3270 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3271 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3272 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3273 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3274 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3275 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3276 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3277 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3278 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3279 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3280 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3281 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3282 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08,
3283 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x84, 0xf8, 0x8b, 0x0f,
3284 0xdb, 0x82, 0xf4, 0xa9, 0x40, 0x0d, 0x03, 0x00,
3285 ];
3286
3287 #[test]
3288 fn decoded_response_cap_counts_decompressed_bytes() {
3289 for cap in [200000u64, 199999] {
3290 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
3291 let base = format!("http://{}", listener.local_addr().unwrap());
3292 let server = std::thread::spawn(move || {
3293 let (mut socket, _) = listener.accept().unwrap();
3294 let mut reader = BufReader::new(socket.try_clone().unwrap());
3295 loop {
3296 let mut line = String::new();
3297 reader.read_line(&mut line).unwrap();
3298 if line == "\r\n" {
3299 break;
3300 }
3301 }
3302 let _ = write!(
3303 socket,
3304 "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
3305 GZIP_BOMB.len()
3306 );
3307 let _ = socket.write_all(&GZIP_BOMB);
3308 });
3309 let mut transport = HostTransport::from_config(
3310 &serde_json::json!({"baseUrl": base, "maxResponseBytes": cap}),
3311 )
3312 .unwrap();
3313 let result = transport.fetch_blob_url(&format!("{base}/signed"));
3314 if cap == 200000 {
3315 let body = result.expect("a cap equal to the decoded size admits the body");
3316 assert_eq!(body.len(), 200000);
3317 } else {
3318 let error = result.unwrap_err();
3319 assert_eq!(error.code, "transport.response_too_large");
3320 assert_eq!(error.details, Some(serde_json::json!({ "limit": cap })));
3321 }
3322 server.join().unwrap();
3323 }
3324 }
3325}