1use std::any::TypeId;
10use std::collections::{BTreeMap, BTreeSet};
11use std::sync::Arc;
12use std::time::{Duration, SystemTime};
13
14use dex_protocol::dex::flow_service_client::FlowServiceClient;
15use dex_protocol::dex::{
16 ActiveStepSearchMode as ProtoSearchMode, AttributeStoreNames, FlowAlreadyStartedOptions,
17 FlowConfig as ProtoFlowConfig, FlowResetStepMethod, FlowResetType, FlowRetryPolicy,
18 FlowStartOptions, FlowStatus as ProtoFlowStatus, FlowTimeoutPolicy as ProtoFlowTimeoutPolicy,
19 GetFlowSummaryRequest, IdReusePolicy as ProtoIdReusePolicy, InvokeRpcRequest,
20 ListStreamMessagesRequest, ReadStreamRequest, ResetFlowRequest, SearchFlowsRequest,
21 SkipTimerRequest, StartFlowRequest, StepDurability as ProtoStepDurability, StopFlowRequest,
22 StopType as ProtoStopType, UpdateFlowConfigRequest, WaitForAttributeRequest,
23 WaitForFlowRequest, WaitForStepCompletionRequest, WorkerTarget as ProtoWorkerTarget,
24 WriteStreamRequest,
25};
26use tokio::runtime::Runtime;
27use tonic::{Code, Request};
28use uuid::Uuid;
29
30mod attribute_reads;
31mod flow_definitions;
32pub(crate) mod flow_service_tls;
33
34pub use flow_service_tls::FlowServiceTlsOptions;
35use flow_service_tls::flow_service_endpoint;
36
37pub use attribute_reads::{AttributeRead, AttributeReads};
38
39use crate::credential::{CredentialChannel, CredentialInterceptor};
40use crate::sdk_error::{FlowTargetRequirement, ServiceError};
41use crate::server_protocol::{NegotiatedServerProtocolVersion, negotiate_server_protocol};
42use crate::stop_flow_options::StopType;
43use crate::time_travel_options::{TimeTravelPoint, TimeTravelStepMethod};
44use crate::value_hydrator::ValueHydrator;
45use crate::value_mapper;
46use crate::wait_options::ClientRequestBudget;
47use crate::worker_dispatcher::{map_flow_timeout_handler_options, map_step_options};
48use crate::{
49 ActiveStepSearchMode, Attribute, AttributeMap, AttributeMatch, BlobCache, ClientOptions, Flow,
50 FlowConfig, FlowErrorType, FlowInfo, FlowResult, FlowStatus, FlowTimeoutPolicy, IdReusePolicy,
51 Registry, RetryPolicy, Rpc, RpcInvokeOptions, SdkError, SdkResult, SearchFlowEntry,
52 SearchFlowsPage, StartFlowOptions, StepCompletion, StepDurability, StepExecutionId,
53 StopFlowOptions, Stream, StreamMessage, StreamMessagesPage, TimeTravelOptions, TimerId, Value,
54 WaitForAttributeOptions, WaitForStepCompletionOptions,
55};
56
57pub struct Client {
93 runtime: Runtime,
94 registry: Registry,
95 service: FlowServiceClient<CredentialChannel>,
96 hydrator: ValueHydrator,
97 negotiated_protocol_version: NegotiatedServerProtocolVersion,
98 protocol_negotiation: tokio::sync::Mutex<Option<Result<u32, String>>>,
99}
100
101impl Client {
102 pub fn new(registry: Registry, blob_cache: Arc<BlobCache>, options: ClientOptions) -> Self {
110 Self::try_new(registry, blob_cache, options)
111 .unwrap_or_else(|error| panic!("cannot create Dex Client: {error}"))
112 }
113
114 pub fn try_new(
122 registry: Registry,
123 blob_cache: Arc<BlobCache>,
124 options: ClientOptions,
125 ) -> SdkResult<Self> {
126 let runtime = Runtime::new().map_err(service_error)?;
127 let endpoint = flow_service_endpoint(
128 options.server_address_value(),
129 options.flow_service_tls_value(),
130 "client",
131 )?;
132 let service = {
133 let _runtime_guard = runtime.enter();
134 FlowServiceClient::new(CredentialChannel::new(
135 endpoint.connect_lazy(),
136 CredentialInterceptor::new(options.credential_value()),
137 ))
138 };
139 let hydrator = ValueHydrator::new(service.clone(), blob_cache);
140 Ok(Self {
141 runtime,
142 registry,
143 service,
144 hydrator,
145 negotiated_protocol_version: NegotiatedServerProtocolVersion::default(),
146 protocol_negotiation: tokio::sync::Mutex::new(None),
147 })
148 }
149
150 pub fn start_flow<SomeFlow: Flow>(
175 &self,
176 flow: &SomeFlow,
177 flow_id: &str,
178 input: SomeFlow::StartInput,
179 ) -> SdkResult<String> {
180 self.start_flow_with_options(flow, flow_id, input, StartFlowOptions::new())
181 }
182
183 pub fn start_flow_with_options<SomeFlow: Flow>(
198 &self,
199 flow: &SomeFlow,
200 flow_id: &str,
201 input: SomeFlow::StartInput,
202 options: StartFlowOptions,
203 ) -> SdkResult<String> {
204 require_name(flow_id, "Flow ID")?;
205 let registered = self.registry.flow(flow.flow_type())?;
206 if registered.type_id != TypeId::of::<SomeFlow>() {
207 return Err(SdkError::FlowDefinition {
208 message: format!(
209 "Flow {} does not match the registered Rust type",
210 flow.flow_type()
211 ),
212 });
213 }
214 let (start_step_type, step_input, step_options) = match ®istered.start_step {
215 Some(step) => (
216 step.name.to_string(),
217 Some(value_mapper::encode(&input)?),
218 Some(
219 map_step_options(
220 registered,
221 registered
222 .handler
223 .step_options(step.name)
224 .map_err(sdk_handler_error)?,
225 )
226 .map_err(sdk_handler_error)?,
227 ),
228 ),
229 None => (String::new(), None, None),
230 };
231 let flow_timeout_seconds = optional_seconds(options.timeout)?;
232 let flow_timeout_policy =
233 map_flow_timeout_policy(registered, flow_timeout_seconds, options.timeout_policy)?;
234 let request = StartFlowRequest {
235 flow_id: flow_id.to_string(),
236 flow_type: registered.name.to_string(),
237 flow_timeout_seconds,
238 flow_timeout_policy,
239 start_step_type,
240 step_input,
241 step_options,
242 flow_start_options: Some(self.map_start_options(
243 registered,
244 &options,
245 flow_timeout_seconds,
246 flow_timeout_policy,
247 )?),
248 request_id: options
249 .request_id
250 .clone()
251 .unwrap_or_else(|| Uuid::new_v4().to_string()),
252 };
253 self.require_server_protocol("start_flow")?;
254 let mut service = self.service.clone();
255 self.runtime.block_on(async {
256 service
257 .start_flow(request)
258 .await
259 .map(|response| response.into_inner().run_id)
260 .map_err(|status| {
261 SdkError::from_status(
262 status,
263 "start_flow",
264 Some(flow_id),
265 FlowTargetRequirement::None,
266 )
267 })
268 })
269 }
270
271 pub fn invoke_rpc<Input: Value, Output: Value>(
299 &self,
300 flow_id: &str,
301 rpc: Rpc<Input, Output>,
302 input: Input,
303 ) -> SdkResult<Output> {
304 self.do_invoke_rpc(flow_id, rpc.name(), &input, &RpcInvokeOptions::new())
305 }
306
307 pub fn invoke_rpc_with_options<Input: Value, Output: Value>(
314 &self,
315 flow_id: &str,
316 rpc: Rpc<Input, Output>,
317 input: Input,
318 options: RpcInvokeOptions,
319 ) -> SdkResult<Output> {
320 self.do_invoke_rpc(flow_id, rpc.name(), &input, &options)
321 }
322
323 pub fn invoke_rpc_without_input<Output: Value>(
327 &self,
328 flow_id: &str,
329 rpc: Rpc<(), Output>,
330 ) -> SdkResult<Output> {
331 self.do_invoke_rpc(flow_id, rpc.name(), &(), &RpcInvokeOptions::new())
332 }
333
334 pub fn invoke_rpc_without_input_with_options<Output: Value>(
340 &self,
341 flow_id: &str,
342 rpc: Rpc<(), Output>,
343 options: RpcInvokeOptions,
344 ) -> SdkResult<Output> {
345 self.do_invoke_rpc(flow_id, rpc.name(), &(), &options)
346 }
347
348 pub fn write_stream<T: Value>(
358 &self,
359 flow_id: &str,
360 stream: &Stream<T>,
361 source: &str,
362 value: T,
363 ) -> SdkResult<()> {
364 require_name(flow_id, "Flow ID")?;
365 require_name(source, "Stream source")?;
366 let flow_type = self.registry.flow_for_stream(stream)?.name.to_string();
367 let request = WriteStreamRequest {
368 flow_id: flow_id.to_string(),
369 flow_type,
370 stream_name: stream.name().to_string(),
371 stream_capacity_bytes: stream.stream_capacity_bytes(),
372 value: Some(value_mapper::encode(&value)?),
373 source: source.to_string(),
374 };
375 self.call_empty(
376 "write_stream",
377 Some(flow_id),
378 FlowTargetRequirement::None,
379 |mut service| async move { service.write_stream(request).await },
380 )
381 }
382
383 pub fn read_stream<T: Value>(
387 &self,
388 flow_id: &str,
389 stream: &Stream<T>,
390 resume_token: &str,
391 ) -> SdkResult<StreamMessage<T>> {
392 self.read_stream_result(flow_id, stream, resume_token, None)
393 }
394
395 pub fn read_stream_with_timeout<T: Value>(
402 &self,
403 flow_id: &str,
404 stream: &Stream<T>,
405 resume_token: &str,
406 timeout: Duration,
407 ) -> SdkResult<StreamMessage<T>> {
408 self.read_stream_result(flow_id, stream, resume_token, Some(timeout))
409 }
410
411 pub fn list_stream_messages<T: Value>(
431 &self,
432 flow_id: &str,
433 stream: &Stream<T>,
434 page_size: i32,
435 before_page_token: &str,
436 ) -> SdkResult<StreamMessagesPage<T>> {
437 require_name(flow_id, "Flow ID")?;
438 if page_size < 1 {
439 return Err(invalid("Stream page size must be positive"));
440 }
441 let flow_type = self.registry.flow_for_stream(stream)?.name.to_string();
442 let request = ListStreamMessagesRequest {
443 flow_id: flow_id.to_string(),
444 flow_type,
445 stream_name: stream.name().to_string(),
446 page_size,
447 before_page_token: before_page_token.to_string(),
448 };
449 self.require_server_protocol("list_stream_messages")?;
450 let mut service = self.service.clone();
451 let response = self.runtime.block_on(async {
452 service
453 .list_stream_messages(request)
454 .await
455 .map(|response| response.into_inner())
456 .map_err(|status| {
457 SdkError::from_status(
458 status,
459 "list_stream_messages",
460 Some(flow_id),
461 FlowTargetRequirement::None,
462 )
463 })
464 })?;
465 let messages = response
466 .messages
467 .into_iter()
468 .map(|message| {
469 if message.resume_token.is_empty() {
470 return Err(invalid("ListStreamMessages returned an empty resume token"));
471 }
472 let value = message
473 .value
474 .ok_or_else(|| invalid("ListStreamMessages omitted a Stream message Value"))?;
475 let created_time = message.created_time.map(timestamp).ok_or_else(|| {
476 invalid("ListStreamMessages omitted a Stream message creation time")
477 })?;
478 Ok(StreamMessage {
479 value: value_mapper::decode(&value)?,
480 resume_token: message.resume_token,
481 created_time,
482 source: message.source,
483 })
484 })
485 .collect::<SdkResult<Vec<_>>>()?;
486 Ok(StreamMessagesPage {
487 messages,
488 next_page_token: response.next_page_token,
489 })
490 }
491
492 pub fn wait_for_flow(&self, flow_id: &str) -> SdkResult<FlowResult> {
499 self.wait_for_flow_result(flow_id, None)
500 }
501
502 pub fn wait_for_flow_with_timeout(
513 &self,
514 flow_id: &str,
515 timeout: Duration,
516 ) -> SdkResult<FlowResult> {
517 self.wait_for_flow_result(flow_id, Some(timeout))
518 }
519
520 pub fn describe_flow(&self, flow_id: &str) -> SdkResult<FlowInfo> {
526 self.require_server_protocol("describe_flow")?;
527 let mut service = self.service.clone();
528 let response = self.runtime.block_on(async {
529 service
530 .get_flow_summary(GetFlowSummaryRequest {
531 flow_id: flow_id.to_string(),
532 run_id: String::new(),
533 })
534 .await
535 .map(|response| response.into_inner())
536 .map_err(|status| {
537 SdkError::from_status(
538 status,
539 "describe_flow",
540 Some(flow_id),
541 FlowTargetRequirement::Existing,
542 )
543 })
544 })?;
545 let execution = response
546 .flow_run_id
547 .ok_or_else(|| service_error("GetFlowSummary omitted FlowRunID"))?;
548 Ok(FlowInfo {
549 flow_id: execution.flow_id,
550 run_id: execution.run_id,
551 flow_type: response.flow_type,
552 status: map_flow_status(response.flow_status)?,
553 started_at: response
554 .start_time
555 .map_or(SystemTime::UNIX_EPOCH, timestamp),
556 })
557 }
558
559 pub fn search_flows(&self, query: &str, page_size: i32) -> SdkResult<SearchFlowsPage> {
564 self.search_flows_page(query, page_size, "")
565 }
566
567 pub fn search_flows_page(
572 &self,
573 query: &str,
574 page_size: i32,
575 next_page_token: &str,
576 ) -> SdkResult<SearchFlowsPage> {
577 if page_size < 0 {
578 return Err(invalid("search page size must not be negative"));
579 }
580 self.require_server_protocol("search_flows")?;
581 let mut service = self.service.clone();
582 self.runtime.block_on(async {
583 let response = service
584 .search_flows(SearchFlowsRequest {
585 query: query.to_string(),
586 page_size,
587 next_page_token: next_page_token.to_string(),
588 })
589 .await
590 .map(|response| response.into_inner())
591 .map_err(|status| {
592 SdkError::from_status(status, "search_flows", None, FlowTargetRequirement::None)
593 })?;
594 let values = response
595 .flow_runs
596 .iter()
597 .flat_map(|entry| {
598 entry.indexed_attributes.iter().map(|attribute| {
599 attribute
600 .value
601 .clone()
602 .map(|value| (entry.flow_id.clone(), value))
603 .ok_or_else(|| invalid("Indexed Attribute has no Value"))
604 })
605 })
606 .collect::<SdkResult<Vec<_>>>()?;
607 let mut values = self.hydrator.hydrate_for_flows(values).await?.into_iter();
608 let mut flows = Vec::with_capacity(response.flow_runs.len());
609 for entry in response.flow_runs {
610 let mut indexed_attributes = BTreeMap::new();
611 for attribute in entry.indexed_attributes {
612 let value = values
613 .next()
614 .ok_or_else(|| invalid("Indexed Attribute hydration count mismatch"))?;
615 indexed_attributes.insert(attribute.key, value_mapper::decode_untyped(&value)?);
616 }
617 flows.push(SearchFlowEntry {
618 flow_id: entry.flow_id,
619 run_id: entry.run_id,
620 flow_type: entry.flow_type,
621 status: map_flow_status(entry.flow_status)?,
622 started_at: entry.start_time.map(timestamp),
623 closed_at: entry.close_time.map(timestamp),
624 indexed_attributes,
625 });
626 }
627 Ok(SearchFlowsPage {
628 flows,
629 next_page_token: response.next_page_token,
630 })
631 })
632 }
633
634 pub fn stop_flow(&self, flow_id: &str, options: StopFlowOptions) -> SdkResult<()> {
643 let stop_type = match options.stop_type {
644 StopType::Cancel => ProtoStopType::Cancel,
645 StopType::Terminate => ProtoStopType::Terminate,
646 StopType::Fail => ProtoStopType::Fail,
647 };
648 self.call_empty(
649 "stop_flow",
650 Some(flow_id),
651 FlowTargetRequirement::Active,
652 |mut service| async move {
653 service
654 .stop_flow(StopFlowRequest {
655 flow_id: flow_id.to_string(),
656 run_id: String::new(),
657 reason: options.reason.unwrap_or_default(),
658 stop_type: stop_type as i32,
659 })
660 .await
661 },
662 )
663 }
664
665 pub fn time_travel(&self, flow_id: &str, options: TimeTravelOptions) -> SdkResult<String> {
679 let mut request = ResetFlowRequest {
680 flow_id: flow_id.to_string(),
681 reset_type: FlowResetType::Unspecified as i32,
682 reason: options.reason.unwrap_or_default(),
683 skip_writes_reapply: options.skip_writes_reapply,
684 ..Default::default()
685 };
686 match options.point {
687 TimeTravelPoint::Beginning => request.reset_type = FlowResetType::Beginning as i32,
688 TimeTravelPoint::HistoryEventTime(time) => {
689 request.reset_type = FlowResetType::HistoryEventTime as i32;
690 request.history_event_time = rfc3339(time)?;
691 }
692 TimeTravelPoint::StepType(step_type) => {
693 request.reset_type = FlowResetType::StepType as i32;
694 request.step_type = step_type.to_string();
695 }
696 TimeTravelPoint::StepExecution(execution, method) => {
697 request.reset_type = FlowResetType::StepExecutionId as i32;
698 request.step_execution_id =
699 format!("{}-{}", execution.step_type, execution.execution_number);
700 request.step_method = match method {
701 TimeTravelStepMethod::WaitFor => FlowResetStepMethod::WaitFor as i32,
702 TimeTravelStepMethod::Execute => FlowResetStepMethod::Execute as i32,
703 };
704 }
705 }
706 self.require_server_protocol("time_travel")?;
707 let mut service = self.service.clone();
708 self.runtime.block_on(async {
709 service
710 .reset_flow(request)
711 .await
712 .map(|response| response.into_inner().run_id)
713 .map_err(|status| {
714 SdkError::from_status(
715 status,
716 "time_travel",
717 Some(flow_id),
718 FlowTargetRequirement::Existing,
719 )
720 })
721 })
722 }
723
724 pub fn skip_timer(
731 &self,
732 flow_id: &str,
733 step_execution: StepExecutionId,
734 timer: TimerId,
735 ) -> SdkResult<()> {
736 let (timer_condition_id, timer_condition_index) = match timer {
737 TimerId::ConditionId(id) => (id, None),
738 TimerId::ConditionIndex(index) => (
739 String::new(),
740 Some(i32::try_from(index).map_err(|_| invalid("timer index exceeds int32"))?),
741 ),
742 };
743 self.call_empty(
744 "skip_timer",
745 Some(flow_id),
746 FlowTargetRequirement::Active,
747 |mut service| async move {
748 service
749 .skip_timer(SkipTimerRequest {
750 flow_id: flow_id.to_string(),
751 run_id: String::new(),
752 step_execution_id: format!(
753 "{}-{}",
754 step_execution.step_type, step_execution.execution_number
755 ),
756 timer_condition_id,
757 timer_condition_index,
758 })
759 .await
760 },
761 )
762 }
763
764 pub fn wait_for_step_completion(
777 &self,
778 flow_id: &str,
779 step_execution: StepExecutionId,
780 options: WaitForStepCompletionOptions,
781 ) -> SdkResult<()> {
782 let request_budget = ClientRequestBudget::new(options.request_timeout)?;
783 self.require_server_protocol_within_budget(
784 "wait_for_step_completion",
785 flow_id,
786 &request_budget,
787 )?;
788 loop {
789 let request_attempt =
790 request_budget.next_attempt("wait_for_step_completion", flow_id)?;
791 let request_flow_id = flow_id.to_string();
792 let request_step_type = step_execution.step_type.to_string();
793 let request_step_execution_number = step_execution.execution_number.to_string();
794 let request_id = options.request_id.clone();
795 let result = self.call_empty(
796 "wait_for_step_completion",
797 Some(flow_id),
798 FlowTargetRequirement::Active,
799 |mut service| async move {
800 let mut request = Request::new(WaitForStepCompletionRequest {
801 flow_id: request_flow_id,
802 step_type: request_step_type,
803 step_execution_number: request_step_execution_number,
804 request_timeout_seconds: request_attempt.request_timeout_seconds,
805 request_id,
806 });
807 if let Some(transport_timeout) = request_attempt.transport_timeout {
808 request.set_timeout(transport_timeout);
809 }
810 service.wait_for_step_completion(request).await
811 },
812 );
813 match result {
814 Err(SdkError::LongPollTimeout { .. }) => continue,
815 Err(error)
816 if error.service_error().is_some_and(|service| {
817 matches!(service.code(), Code::DeadlineExceeded | Code::Cancelled)
818 }) && request_budget.has_deadline() =>
819 {
820 return Err(SdkError::request_timeout(
821 "wait_for_step_completion",
822 flow_id,
823 ));
824 }
825 result => return result,
826 }
827 }
828 }
829
830 pub fn wait_for_attribute_match<T: Value>(
843 &self,
844 flow_id: &str,
845 attribute: &Attribute<T>,
846 attribute_match: AttributeMatch<T>,
847 options: WaitForAttributeOptions,
848 ) -> SdkResult<T> {
849 require_waitable_attribute(attribute.name(), attribute.is_hard_delete_on_null())?;
850 self.wait_for_attribute_value(flow_id, attribute.name(), &attribute_match, options)
851 }
852
853 pub fn wait_for_attribute_map_instance_match<T: Value>(
860 &self,
861 flow_id: &str,
862 attribute: &AttributeMap<T>,
863 instance: &str,
864 attribute_match: AttributeMatch<T>,
865 options: WaitForAttributeOptions,
866 ) -> SdkResult<T> {
867 require_waitable_attribute(attribute.name(), attribute.is_hard_delete_on_null())?;
868 self.wait_for_attribute_value(
869 flow_id,
870 &map_physical_name(attribute.name(), instance)?,
871 &attribute_match,
872 options,
873 )
874 }
875
876 fn wait_for_attribute_value<T: Value>(
877 &self,
878 flow_id: &str,
879 key: &str,
880 attribute_match: &AttributeMatch<T>,
881 options: WaitForAttributeOptions,
882 ) -> SdkResult<T> {
883 let mut encoded_match = attribute_match.encode()?;
884 encoded_match.key = key.to_string();
885 let request_budget = ClientRequestBudget::new(options.request_timeout)?;
886 self.require_server_protocol_within_budget(
887 "wait_for_attribute_match",
888 flow_id,
889 &request_budget,
890 )?;
891 let response = loop {
892 let request_attempt =
893 request_budget.next_attempt("wait_for_attribute_match", flow_id)?;
894 let mut service = self.service.clone();
895 let mut request = Request::new(WaitForAttributeRequest {
896 flow_id: flow_id.to_string(),
897 r#match: Some(encoded_match.clone()),
898 request_timeout_seconds: request_attempt.request_timeout_seconds,
899 request_id: options.request_id.clone(),
900 });
901 if let Some(transport_timeout) = request_attempt.transport_timeout {
902 request.set_timeout(transport_timeout);
903 }
904 let result = self
905 .runtime
906 .block_on(service.wait_for_attribute(request))
907 .map_err(|status| {
908 SdkError::from_status(
909 status,
910 "wait_for_attribute_match",
911 Some(flow_id),
912 FlowTargetRequirement::Active,
913 )
914 });
915 match result {
916 Ok(response) => break response.into_inner(),
917 Err(SdkError::LongPollTimeout { .. }) => continue,
918 Err(error)
919 if error.service_error().is_some_and(|service| {
920 matches!(service.code(), Code::DeadlineExceeded | Code::Cancelled)
921 }) && request_budget.has_deadline() =>
922 {
923 return Err(SdkError::request_timeout(
924 "wait_for_attribute_match",
925 flow_id,
926 ));
927 }
928 Err(error) => return Err(error),
929 }
930 };
931 let matched_value = response.matched_value.ok_or_else(|| SdkError::Service {
932 service: ServiceError::local(
933 "wait_for_attribute_match",
934 "WaitForAttribute response is incomplete",
935 ),
936 })?;
937 value_mapper::decode(&matched_value)
938 }
939
940 pub fn update_flow_config(&self, flow_id: &str, config: FlowConfig) -> SdkResult<()> {
945 let flow_config = self.map_flow_config(Some(&config))?;
946 self.call_empty(
947 "update_flow_config",
948 Some(flow_id),
949 FlowTargetRequirement::Active,
950 |mut service| async move {
951 service
952 .update_flow_config(UpdateFlowConfigRequest {
953 flow_id: flow_id.to_string(),
954 run_id: String::new(),
955 flow_config: Some(flow_config),
956 })
957 .await
958 },
959 )
960 }
961
962 pub fn health_check(&self) -> SdkResult<()> {
966 self.call_empty(
967 "health_check",
968 None,
969 FlowTargetRequirement::None,
970 |mut service| async move { service.health_check(()).await },
971 )
972 }
973
974 fn do_invoke_rpc<Input: Value, Output: Value>(
975 &self,
976 flow_id: &str,
977 rpc_name: &str,
978 input: &Input,
979 invoke_options: &RpcInvokeOptions,
980 ) -> SdkResult<Output> {
981 let (flow, rpc) = self.registry.rpc_with_flow(rpc_name)?;
982 let mut lock_attribute_keys = rpc
983 .locks
984 .iter()
985 .map(|lock| lock.physical_name())
986 .collect::<BTreeSet<_>>();
987 for lock in &invoke_options.lock_attribute_map_instances {
988 let Some(instance) = lock.instance() else {
989 return Err(SdkError::FlowDefinition {
990 message: "RpcInvokeOptions lock must target an AttributeMap instance".into(),
991 });
992 };
993 require_rpc_invoke_definition(
994 flow,
995 lock.attribute_name(),
996 crate::persistence::PersistenceKind::AttributeMap,
997 )?;
998 crate::registry::validate_map_instance(instance).map_err(invalid)?;
999 lock_attribute_keys.insert(lock.physical_name());
1000 }
1001 let mut load_attribute_map_instances = rpc
1002 .load_attribute_maps
1003 .iter()
1004 .map(|load| map_load_name(&load.name, load.instance.as_deref()))
1005 .collect::<BTreeSet<_>>();
1006 for load in &invoke_options.load_attribute_map_instances {
1007 let instance = require_rpc_invoke_map_load(
1008 flow,
1009 &load.name,
1010 load.instance.as_deref(),
1011 crate::persistence::PersistenceKind::AttributeMap,
1012 )?;
1013 load_attribute_map_instances
1014 .insert(crate::registry::physical_name(&load.name, instance));
1015 }
1016 let mut load_channel_names = rpc
1017 .load_channels
1018 .iter()
1019 .map(|load| load.name.clone())
1020 .collect::<Vec<_>>();
1021 load_channel_names.sort();
1022 let mut load_channel_map_instances = rpc
1023 .load_channel_maps
1024 .iter()
1025 .map(|load| map_load_name(&load.name, load.instance.as_deref()))
1026 .collect::<BTreeSet<_>>();
1027 for load in &invoke_options.load_channel_map_instances {
1028 let instance = require_rpc_invoke_map_load(
1029 flow,
1030 &load.name,
1031 load.instance.as_deref(),
1032 crate::persistence::PersistenceKind::ChannelMap,
1033 )?;
1034 load_channel_map_instances.insert(crate::registry::physical_name(&load.name, instance));
1035 }
1036 let request = InvokeRpcRequest {
1037 flow_id: flow_id.to_string(),
1038 run_id: String::new(),
1039 rpc_name: rpc_name.to_string(),
1040 input: Some(value_mapper::encode(input)?),
1041 timeout_seconds: optional_seconds(rpc.timeout)?,
1042 lock_attribute_keys: lock_attribute_keys.into_iter().collect(),
1043 request_id: Uuid::new_v4().to_string(),
1044 is_transactional: rpc.is_transactional,
1045 load_attribute_map_instances: load_attribute_map_instances.into_iter().collect(),
1046 load_channel_names,
1047 load_channel_map_instances: load_channel_map_instances.into_iter().collect(),
1048 };
1049 self.require_server_protocol("invoke_rpc")?;
1050 let mut service = self.service.clone();
1051 let output = self.runtime.block_on(async {
1052 service
1053 .invoke_rpc(request)
1054 .await
1055 .map(|response| response.into_inner().output)
1056 .map_err(|status| {
1057 SdkError::from_status(
1058 status,
1059 "invoke_rpc",
1060 Some(flow_id),
1061 FlowTargetRequirement::Active,
1062 )
1063 })
1064 })?;
1065 let output = output.ok_or_else(|| invalid("InvokeRPC omitted output"))?;
1066 let output = self
1067 .runtime
1068 .block_on(self.hydrator.hydrate(flow_id, output))?;
1069 value_mapper::decode(&output)
1070 }
1071
1072 fn read_stream_result<T: Value>(
1073 &self,
1074 flow_id: &str,
1075 stream: &Stream<T>,
1076 resume_token: &str,
1077 timeout: Option<Duration>,
1078 ) -> SdkResult<StreamMessage<T>> {
1079 require_name(flow_id, "Flow ID")?;
1080 let flow_type = self.registry.flow_for_stream(stream)?.name.to_string();
1081 let wait_time_seconds = optional_seconds(timeout)?;
1082 let request = ReadStreamRequest {
1083 flow_id: flow_id.to_string(),
1084 flow_type,
1085 stream_name: stream.name().to_string(),
1086 resume_token: resume_token.to_string(),
1087 wait_time_seconds,
1088 };
1089 self.require_server_protocol("read_stream")?;
1090 let mut service = self.service.clone();
1091 let response = self.runtime.block_on(async {
1092 service
1093 .read_stream(request)
1094 .await
1095 .map(|response| response.into_inner())
1096 .map_err(|status| {
1097 SdkError::from_status(
1098 status,
1099 "read_stream",
1100 Some(flow_id),
1101 FlowTargetRequirement::None,
1102 )
1103 })
1104 })?;
1105 let message = response
1106 .message
1107 .ok_or_else(|| invalid("ReadStream omitted its message"))?;
1108 if message.resume_token.is_empty() {
1109 return Err(invalid("ReadStream returned an empty resume token"));
1110 }
1111 let value = message
1112 .value
1113 .ok_or_else(|| invalid("ReadStream omitted its Value"))?;
1114 let created_time = message
1115 .created_time
1116 .map(timestamp)
1117 .ok_or_else(|| invalid("ReadStream omitted its creation time"))?;
1118 Ok(StreamMessage {
1119 value: value_mapper::decode(&value)?,
1120 resume_token: message.resume_token,
1121 created_time,
1122 source: message.source,
1123 })
1124 }
1125
1126 fn wait_for_flow_result(
1127 &self,
1128 flow_id: &str,
1129 timeout: Option<Duration>,
1130 ) -> SdkResult<FlowResult> {
1131 self.require_server_protocol("wait_for_flow")?;
1132 let mut service = self.service.clone();
1133 let response = self.runtime.block_on(async {
1134 service
1135 .wait_for_flow(WaitForFlowRequest {
1136 flow_id: flow_id.to_string(),
1137 run_id: String::new(),
1138 needs_results: true,
1139 wait_time_seconds: optional_seconds(timeout)?,
1140 })
1141 .await
1142 .map(|response| response.into_inner())
1143 .map_err(|status| {
1144 SdkError::from_status(
1145 status,
1146 "wait_for_flow",
1147 Some(flow_id),
1148 FlowTargetRequirement::Existing,
1149 )
1150 })
1151 });
1152 let response = response?;
1153 let mut outputs = Vec::with_capacity(response.results.len());
1154 for result in &response.results {
1155 outputs.push(result.completed_step_output.clone().ok_or_else(|| {
1156 SdkError::ValueMapping {
1157 message: "Step completion output is required".to_string(),
1158 }
1159 })?);
1160 }
1161 let outputs = self
1162 .runtime
1163 .block_on(self.hydrator.hydrate_all(flow_id, outputs))?;
1164 let completions = response
1165 .results
1166 .into_iter()
1167 .zip(outputs)
1168 .map(|(result, output)| {
1169 StepCompletion::new(
1170 result.completed_step_type,
1171 result.completed_step_execution_id,
1172 output,
1173 )
1174 })
1175 .collect();
1176 Ok(FlowResult::new(
1177 map_flow_status(response.flow_status)?,
1178 map_flow_error_type(response.error_type),
1179 (!response.error_message.is_empty()).then_some(response.error_message),
1180 completions,
1181 ))
1182 }
1183
1184 fn map_start_options(
1185 &self,
1186 flow: &crate::registry::RegisteredFlow,
1187 options: &StartFlowOptions,
1188 flow_timeout_seconds: i32,
1189 flow_timeout_policy: i32,
1190 ) -> SdkResult<FlowStartOptions> {
1191 let attributes = options
1192 .attributes
1193 .iter()
1194 .map(|attribute| {
1195 let definition =
1196 flow.persistence
1197 .get(&attribute.attribute_name)
1198 .ok_or_else(|| {
1199 invalid(format!(
1200 "initial Attribute is not registered: {}",
1201 attribute.key
1202 ))
1203 })?;
1204 attribute.encode_attribute_write(definition)
1205 })
1206 .collect::<SdkResult<Vec<_>>>()?;
1207 if let Some(write) = attributes.iter().find(|write| write.is_hard_delete_on_null) {
1208 self.require_hard_delete_attribute_support("start_flow", &write.key)?;
1209 }
1210 let config = options.config_override.as_ref();
1211 let flow_config_override = Some(map_pull_flow_config(config)?);
1212 #[expect(
1213 clippy::needless_update,
1214 reason = "The test preserves forward-compatible construction of SDK options."
1215 )]
1216 let flow_start_options = FlowStartOptions {
1217 id_reuse_policy: match options.id_reuse_policy {
1218 IdReusePolicy::Default => ProtoIdReusePolicy::Unspecified,
1219 IdReusePolicy::AllowIfPreviousFailed => {
1220 ProtoIdReusePolicy::AllowIfPreviousExistsAbnormally
1221 }
1222 IdReusePolicy::AllowIfNotRunning => ProtoIdReusePolicy::AllowIfNoRunning,
1223 IdReusePolicy::Disallow => ProtoIdReusePolicy::DisallowReuse,
1224 IdReusePolicy::TerminateIfRunning => ProtoIdReusePolicy::AllowTerminateIfRunning,
1225 } as i32,
1226 flow_start_delay_seconds: optional_seconds(options.start_delay)?,
1227 retry_policy: options
1228 .retry_policy
1229 .clone()
1230 .map(map_flow_retry)
1231 .transpose()?,
1232 attributes,
1233 flow_config_override,
1234 flow_already_started_options: Some(FlowAlreadyStartedOptions {
1235 ignore_already_started_error: options.ignore_already_started,
1236 }),
1237 timeout_handler_options: map_flow_timeout_handler_options(
1238 flow,
1239 flow_timeout_seconds,
1240 flow_timeout_policy,
1241 options.timeout_handler_options.as_ref(),
1242 )
1243 .map_err(sdk_handler_error)?,
1244 ..Default::default()
1245 };
1246 Ok(flow_start_options)
1247 }
1248
1249 fn require_server_protocol(&self, operation: &'static str) -> SdkResult<()> {
1250 self.runtime
1251 .block_on(self.negotiate_server_protocol_once(operation))
1252 }
1253
1254 fn require_server_protocol_within_budget(
1257 &self,
1258 operation: &'static str,
1259 flow_id: &str,
1260 request_budget: &ClientRequestBudget,
1261 ) -> SdkResult<()> {
1262 let Some(deadline) = request_budget.deadline() else {
1263 return self.require_server_protocol(operation);
1264 };
1265 self.runtime
1266 .block_on(async {
1267 tokio::time::timeout_at(
1268 tokio::time::Instant::from_std(deadline),
1269 self.negotiate_server_protocol_once(operation),
1270 )
1271 .await
1272 })
1273 .unwrap_or_else(|_| Err(SdkError::request_timeout(operation, flow_id)))
1274 }
1275
1276 async fn negotiate_server_protocol_once(&self, operation: &'static str) -> SdkResult<()> {
1277 let mut cached = self.protocol_negotiation.lock().await;
1278 if cached.is_none() {
1279 let mut service = self.service.clone();
1280 let server_info = service
1281 .get_server_info(())
1282 .await
1283 .map_err(|status| {
1284 SdkError::from_status(status, operation, None, FlowTargetRequirement::None)
1285 })?
1286 .into_inner();
1287 *cached = Some(negotiate_server_protocol(&server_info));
1288 }
1289 match cached.as_ref().expect("Server negotiation was cached") {
1290 Ok(version) => {
1291 self.negotiated_protocol_version.store(*version);
1292 Ok(())
1293 }
1294 Err(message) => Err(SdkError::Service {
1295 service: ServiceError::local("server_protocol", message.clone()),
1296 }),
1297 }
1298 }
1299
1300 fn require_hard_delete_attribute_support(
1301 &self,
1302 operation: &'static str,
1303 attribute_key: &str,
1304 ) -> SdkResult<()> {
1305 self.require_server_protocol(operation)?;
1306 self.negotiated_protocol_version
1307 .require_hard_delete_attribute_support(attribute_key)
1308 .map_err(|error| SdkError::Service {
1309 service: ServiceError::local(operation, error),
1310 })
1311 }
1312
1313 fn map_flow_config(&self, config: Option<&FlowConfig>) -> SdkResult<ProtoFlowConfig> {
1314 map_flow_config(config)
1315 }
1316
1317 fn call_empty<Response, Future, Call>(
1318 &self,
1319 operation: &'static str,
1320 flow_id: Option<&str>,
1321 requirement: FlowTargetRequirement,
1322 call: Call,
1323 ) -> SdkResult<()>
1324 where
1325 Call: FnOnce(FlowServiceClient<CredentialChannel>) -> Future,
1326 Future: std::future::Future<Output = Result<tonic::Response<Response>, tonic::Status>>,
1327 {
1328 self.require_server_protocol(operation)?;
1329 let service = self.service.clone();
1330 self.runtime
1331 .block_on(call(service))
1332 .map(|_| ())
1333 .map_err(|status| SdkError::from_status(status, operation, flow_id, requirement))
1334 }
1335}
1336
1337pub(crate) fn map_pull_flow_config(config: Option<&FlowConfig>) -> SdkResult<ProtoFlowConfig> {
1338 Ok(ProtoFlowConfig {
1339 worker_target: Some(ProtoWorkerTarget { is_pull_mode: true }),
1340 ..map_flow_config(config)?
1341 })
1342}
1343
1344pub(crate) fn map_flow_config(config: Option<&FlowConfig>) -> SdkResult<ProtoFlowConfig> {
1345 Ok(ProtoFlowConfig {
1346 active_step_search_mode: config
1347 .and_then(|config| config.active_step_search_mode)
1348 .map(|mode| match mode {
1349 ActiveStepSearchMode::All => ProtoSearchMode::EnabledForAll as i32,
1350 ActiveStepSearchMode::WithWaitFor => {
1351 ProtoSearchMode::EnabledForStepsWithWaitFor as i32
1352 }
1353 ActiveStepSearchMode::Disabled => ProtoSearchMode::Disabled as i32,
1354 }),
1355 step_durability: config
1356 .and_then(|config| config.step_durability)
1357 .map(|durability| {
1358 (match durability {
1359 StepDurability::Default => ProtoStepDurability::Unspecified,
1360 StepDurability::Sync => ProtoStepDurability::Sync,
1361 StepDurability::Async => ProtoStepDurability::Async,
1362 }) as i32
1363 }),
1364 worker_target: None,
1365 attribute_store_names: config.and_then(|config| {
1366 config
1367 .attribute_store_names
1368 .as_ref()
1369 .map(|names| AttributeStoreNames {
1370 names: names.clone(),
1371 })
1372 }),
1373 history_window_events: config
1375 .and_then(|config| config.history_window_events)
1376 .map(i32::try_from)
1377 .transpose()
1378 .map_err(|_| invalid("history window events exceed int32"))?,
1379 delete_expired_history_objects: None,
1381 offload_history: config.and_then(|config| config.offload_history),
1382 })
1383}
1384
1385pub(crate) fn map_flow_retry(retry: RetryPolicy) -> SdkResult<FlowRetryPolicy> {
1386 Ok(FlowRetryPolicy {
1387 initial_interval_seconds: optional_seconds(retry.initial_interval)?,
1388 backoff_coefficient: retry.backoff_coefficient.unwrap_or_default() as f32,
1389 maximum_interval_seconds: optional_seconds(retry.maximum_interval)?,
1390 maximum_attempts: i32::try_from(retry.maximum_attempts.unwrap_or_default())
1391 .map_err(|_| invalid("maximum attempts exceed int32"))?,
1392 })
1393}
1394
1395pub(crate) fn map_flow_status(status: i32) -> SdkResult<FlowStatus> {
1396 match ProtoFlowStatus::try_from(status).ok() {
1397 Some(ProtoFlowStatus::Running) => Ok(FlowStatus::Running),
1398 Some(ProtoFlowStatus::Completed) => Ok(FlowStatus::Completed),
1399 Some(ProtoFlowStatus::Failed) => Ok(FlowStatus::Failed),
1400 Some(ProtoFlowStatus::ServerSideTimeoutInternalOnly) => {
1401 Ok(FlowStatus::ServerSideTimeoutInternalOnly)
1402 }
1403 Some(ProtoFlowStatus::Terminated) => Ok(FlowStatus::Terminated),
1404 Some(ProtoFlowStatus::Canceled) => Ok(FlowStatus::Canceled),
1405 _ => Err(invalid(format!("unknown Flow status {status}"))),
1406 }
1407}
1408
1409pub(crate) fn map_flow_error_type(error_type: i32) -> Option<FlowErrorType> {
1410 use dex_protocol::dex::FlowErrorType as ProtoFlowErrorType;
1411
1412 match ProtoFlowErrorType::try_from(error_type).ok() {
1413 Some(ProtoFlowErrorType::StepDecisionFailingFlow) => {
1414 Some(FlowErrorType::StepDecisionFailed)
1415 }
1416 Some(ProtoFlowErrorType::ClientApiFailingFlow) => Some(FlowErrorType::ClientApiFailed),
1417 Some(ProtoFlowErrorType::WorkerApiFail) => Some(FlowErrorType::WorkerApiFailed),
1418 Some(ProtoFlowErrorType::InvalidUserFlowCode) => Some(FlowErrorType::InvalidUserFlowCode),
1419 Some(ProtoFlowErrorType::Internal) => Some(FlowErrorType::Internal),
1420 Some(ProtoFlowErrorType::FlowTimeout) => Some(FlowErrorType::FlowTimeout),
1421 _ => None,
1422 }
1423}
1424
1425pub(crate) fn map_flow_timeout_policy(
1426 flow: &crate::registry::RegisteredFlow,
1427 timeout_seconds: i32,
1428 policy: FlowTimeoutPolicy,
1429) -> SdkResult<i32> {
1430 if timeout_seconds == 0 {
1431 if policy != FlowTimeoutPolicy::Default {
1432 return Err(invalid("Flow timeout policy requires a positive timeout"));
1433 }
1434 return Ok(ProtoFlowTimeoutPolicy::Unspecified as i32);
1435 }
1436 let resolved = match policy {
1437 FlowTimeoutPolicy::Default if flow.handler.has_timeout_handler() => {
1438 FlowTimeoutPolicy::Handler
1439 }
1440 FlowTimeoutPolicy::Default => FlowTimeoutPolicy::Fail,
1441 configured => configured,
1442 };
1443 if resolved == FlowTimeoutPolicy::Handler && !flow.handler.has_timeout_handler() {
1444 return Err(invalid(format!(
1445 "Flow {} has no timeout handler",
1446 flow.name
1447 )));
1448 }
1449 Ok(match resolved {
1450 FlowTimeoutPolicy::Default => ProtoFlowTimeoutPolicy::Unspecified,
1451 FlowTimeoutPolicy::Fail => ProtoFlowTimeoutPolicy::Fail,
1452 FlowTimeoutPolicy::Cancel => ProtoFlowTimeoutPolicy::Cancel,
1453 FlowTimeoutPolicy::Handler => ProtoFlowTimeoutPolicy::Handler,
1454 } as i32)
1455}
1456
1457fn timestamp(timestamp: prost_types::Timestamp) -> SystemTime {
1458 if timestamp.seconds >= 0 {
1459 SystemTime::UNIX_EPOCH
1460 + Duration::from_secs(timestamp.seconds as u64)
1461 + Duration::from_nanos(timestamp.nanos.max(0) as u64)
1462 } else {
1463 SystemTime::UNIX_EPOCH - Duration::from_secs(timestamp.seconds.unsigned_abs())
1464 }
1465}
1466
1467fn rfc3339(time: SystemTime) -> SdkResult<String> {
1468 let duration = time
1469 .duration_since(SystemTime::UNIX_EPOCH)
1470 .map_err(service_error)?;
1471 let days = (duration.as_secs() / 86_400) as i64;
1472 let seconds = duration.as_secs() % 86_400;
1473 let (year, month, day) = civil_from_days(days);
1474 let hour = seconds / 3_600;
1475 let minute = seconds % 3_600 / 60;
1476 let second = seconds % 60;
1477 Ok(format!(
1478 "{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}Z"
1479 ))
1480}
1481
1482fn civil_from_days(days: i64) -> (i64, i64, i64) {
1483 let days = days + 719_468;
1484 let era = if days >= 0 { days } else { days - 146_096 } / 146_097;
1485 let day_of_era = days - era * 146_097;
1486 let year_of_era =
1487 (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
1488 let mut year = year_of_era + era * 400;
1489 let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
1490 let month_prime = (5 * day_of_year + 2) / 153;
1491 let day = day_of_year - (153 * month_prime + 2) / 5 + 1;
1492 let month = month_prime + if month_prime < 10 { 3 } else { -9 };
1493 year += i64::from(month <= 2);
1494 (year, month, day)
1495}
1496
1497fn map_load_name(name: &str, instance: Option<&str>) -> String {
1498 match instance {
1499 Some(instance) => crate::registry::physical_name(name, instance),
1500 None => format!("{name}/"),
1501 }
1502}
1503
1504fn require_rpc_invoke_map_load<'a>(
1505 flow: &crate::registry::RegisteredFlow,
1506 name: &str,
1507 instance: Option<&'a str>,
1508 expected_kind: crate::persistence::PersistenceKind,
1509) -> SdkResult<&'a str> {
1510 require_rpc_invoke_definition(flow, name, expected_kind)?;
1511 let instance = instance.ok_or_else(|| SdkError::FlowDefinition {
1512 message: "RpcInvokeOptions requires an exact map instance".to_string(),
1513 })?;
1514 crate::registry::validate_map_instance(instance).map_err(invalid)?;
1515 Ok(instance)
1516}
1517
1518fn require_rpc_invoke_definition(
1519 flow: &crate::registry::RegisteredFlow,
1520 name: &str,
1521 expected_kind: crate::persistence::PersistenceKind,
1522) -> SdkResult<()> {
1523 flow.persistence
1524 .get(name)
1525 .filter(|definition| definition.kind == expected_kind)
1526 .ok_or_else(|| SdkError::FlowDefinition {
1527 message: format!(
1528 "Flow {} does not register the selected map {name}",
1529 flow.name
1530 ),
1531 })?;
1532 Ok(())
1533}
1534
1535fn optional_seconds(duration: Option<Duration>) -> SdkResult<i32> {
1536 duration
1537 .map(seconds32)
1538 .transpose()
1539 .map(Option::unwrap_or_default)
1540}
1541
1542pub(crate) fn seconds32(duration: Duration) -> SdkResult<i32> {
1543 if duration.subsec_nanos() != 0 {
1544 return Err(invalid("Duration must use whole seconds"));
1545 }
1546 i32::try_from(duration.as_secs()).map_err(|_| invalid("Duration exceeds int32"))
1547}
1548
1549fn require_name(value: &str, kind: &str) -> SdkResult<()> {
1550 if value.is_empty() {
1551 Err(invalid(format!("{kind} is required")))
1552 } else {
1553 Ok(())
1554 }
1555}
1556
1557fn map_physical_name(name: &str, instance: &str) -> SdkResult<String> {
1558 crate::registry::validate_map_instance(instance).map_err(invalid)?;
1559 Ok(crate::registry::physical_name(name, instance))
1560}
1561
1562fn require_waitable_attribute(attribute_name: &str, is_hard_delete_on_null: bool) -> SdkResult<()> {
1563 if is_hard_delete_on_null {
1564 return Err(invalid(format!(
1565 "wait_for_attribute_match does not support hard-delete Attribute {attribute_name}"
1566 )));
1567 }
1568 Ok(())
1569}
1570
1571fn sdk_handler_error(error: impl std::fmt::Display) -> SdkError {
1572 invalid(error.to_string())
1573}
1574
1575pub(crate) fn invalid(message: impl Into<String>) -> SdkError {
1576 SdkError::InvalidArgument {
1577 message: message.into(),
1578 }
1579}
1580
1581fn service_error(error: impl std::fmt::Display) -> SdkError {
1582 SdkError::Service {
1583 service: ServiceError::local("client", error.to_string()),
1584 }
1585}
1586
1587#[cfg(test)]
1588mod tests;
1589
1590#[cfg(test)]
1591#[path = "client_pull_routing_tests.rs"]
1592mod pull_routing_tests;