1use crate::RequestOptions;
16use crate::error::QueryError;
17use crate::generated::{CompleteQueryMetadata, QueryMetadata};
18use crate::query::execution::RetryContext;
19use crate::query::retry_policy::JobRetryResult;
20use crate::query::{Result, RowIterator};
21use bytes::Bytes;
22use google_cloud_bigquery_v2::builder::job_service::GetJob;
23use google_cloud_bigquery_v2::client::JobService;
24use google_cloud_bigquery_v2::model::query_response::{Results, ResultsSchema};
25use google_cloud_bigquery_v2::model::{
26 GetQueryResultsRequest, GetQueryResultsResponse, Job, JobReference, QueryResponse,
27};
28use google_cloud_gax::exponential_backoff::ExponentialBackoffBuilder;
29use google_cloud_gax::polling_backoff_policy::PollingBackoffPolicy;
30use google_cloud_gax::polling_state::PollingState;
31use std::collections::VecDeque;
32use std::sync::Arc;
33
34#[derive(Clone, Debug)]
58pub struct Query {
59 pub(crate) job_service: Arc<JobService>,
60 pub(crate) completed: bool,
61 pub(crate) metadata: QueryMetadata,
62 pub(crate) cached_data: Option<CachedData>,
63 pub(crate) page_size: Option<u32>,
64 pub(crate) retry_context: Option<RetryContext>,
65 pub(crate) options: RequestOptions,
66}
67
68#[derive(Clone, Debug)]
69pub(crate) enum CachedData {
70 Rows(VecDeque<wkt::Struct>),
71 #[allow(dead_code)]
72 Arrow {
73 serialized_record_batch: Bytes,
74 serialized_schema: Bytes,
75 },
76}
77
78impl Query {
79 pub(crate) fn from_job(
80 job_service: Arc<JobService>,
81 initial_job: Job,
82 retry_context: Option<&RetryContext>,
83 page_size: Option<u32>,
84 options: RequestOptions,
85 ) -> Self {
86 let completed = initial_job
87 .status
88 .as_ref()
89 .map(|s| s.state == "DONE")
90 .unwrap_or(false);
91 Self {
92 job_service,
93 completed,
94 cached_data: None,
95 metadata: build_query_metadata_from_job(initial_job),
96 retry_context: retry_context.filter(|_| !completed).cloned(),
97 page_size,
98 options,
99 }
100 }
101
102 pub(crate) fn from_query_response(
103 job_service: Arc<JobService>,
104 mut query_response: QueryResponse,
105 retry_context: Option<&RetryContext>,
106 page_size: Option<u32>,
107 options: RequestOptions,
108 ) -> Self {
109 let completed = query_response.job_complete.unwrap_or(false);
110 let cached_data = if let (
111 Some(ResultsSchema::ArrowSchema(schema)),
112 Some(Results::ArrowRecordBatch(results)),
113 ) = (
114 query_response.results_schema.take(),
115 query_response.results.take(),
116 ) {
117 Some(CachedData::Arrow {
118 serialized_record_batch: results.serialized_record_batch,
119 serialized_schema: schema.serialized_schema,
120 })
121 } else {
122 let cached_rows = VecDeque::from(std::mem::take(&mut query_response.rows));
123 Some(CachedData::Rows(cached_rows))
124 };
125 let metadata = QueryMetadata::from(query_response);
126 Self {
127 job_service,
128 completed,
129 cached_data,
130 metadata,
131 retry_context: retry_context.filter(|_| !completed).cloned(),
132 page_size,
133 options,
134 }
135 }
136
137 pub fn metadata(&self) -> &QueryMetadata {
148 &self.metadata
149 }
150
151 pub fn get_job(&self) -> Option<GetJob> {
188 build_get_job(
189 &self.job_service,
190 self.metadata.job_reference.as_ref()?,
191 &self.options,
192 )
193 }
194
195 pub(crate) fn is_dry_run(&self) -> bool {
196 self.metadata
197 .configuration
198 .as_ref()
199 .and_then(|c| c.dry_run)
200 .unwrap_or(false)
201 }
202
203 pub async fn until_done(mut self) -> Result<CompleteQuery> {
231 if self.is_dry_run() {
232 return Err(QueryError::DryRun);
233 }
234
235 loop {
236 let Query {
237 job_service,
238 completed,
239 metadata,
240 cached_data,
241 page_size,
242 retry_context,
243 options,
244 } = self;
245
246 if completed && let Some(cached_data) = cached_data {
247 return Ok(CompleteQuery::from_query_metadata(
248 job_service,
249 metadata,
250 cached_data,
251 page_size,
252 options,
253 ));
254 }
255
256 let job_ref = metadata
257 .job_reference
258 .clone()
259 .expect("query job should have job reference at this point");
260
261 let backoff_policy = Arc::new(
262 ExponentialBackoffBuilder::default()
263 .with_initial_delay(std::time::Duration::from_secs(1))
264 .build()
265 .expect("valid backoff configuration"),
266 );
267
268 match poll_query_results(&job_service, &job_ref, backoff_policy, &options).await {
269 Ok(res) => {
270 return Ok(CompleteQuery::from_get_query_results_response(
271 job_service,
272 &job_ref,
273 res,
274 metadata,
275 page_size,
276 options,
277 ));
278 }
279 Err(err) => {
280 let Some(retry_ctx) = retry_context else {
281 return Err(err);
282 };
283 match retry_ctx.on_error(err) {
284 JobRetryResult::Continue(delay, _) => {
285 self = retry_ctx.reissue(delay).await?;
286 }
287 JobRetryResult::Permanent(e) | JobRetryResult::Exhausted(e) => {
288 return Err(e);
289 }
290 }
291 }
292 }
293 }
294 }
295}
296
297#[derive(Clone, Debug)]
320pub struct CompleteQuery {
321 pub(crate) job_service: Arc<JobService>,
322 pub(crate) job_ref: Option<JobReference>,
323 pub(crate) cached_data: CachedData,
324 pub(crate) page_token: Option<String>,
325 pub(crate) metadata: CompleteQueryMetadata,
326 pub(crate) page_size: Option<u32>,
327 pub(crate) options: RequestOptions,
328}
329
330impl CompleteQuery {
331 pub(crate) fn from_get_query_results_response(
332 job_service: Arc<JobService>,
333 job_ref: &JobReference,
334 mut res: GetQueryResultsResponse,
335 initial_metadata: QueryMetadata,
336 page_size: Option<u32>,
337 options: RequestOptions,
338 ) -> Self {
339 let cached_rows = VecDeque::from(std::mem::take(&mut res.rows));
340 let cached_data = CachedData::Rows(cached_rows);
341 let metadata =
342 build_complete_query_metadata_from_get_query_results(initial_metadata, res, job_ref);
343 Self::from_complete_metadata(
344 job_service,
345 Some(job_ref.clone()),
346 metadata,
347 cached_data,
348 page_size,
349 options,
350 )
351 }
352
353 pub(crate) fn from_query_metadata(
354 job_service: Arc<JobService>,
355 metadata: QueryMetadata,
356 cached_data: CachedData,
357 page_size: Option<u32>,
358 options: RequestOptions,
359 ) -> Self {
360 let job_ref = metadata.job_reference.clone();
361 let metadata = CompleteQueryMetadata::from(metadata);
362 Self::from_complete_metadata(
363 job_service,
364 job_ref,
365 metadata,
366 cached_data,
367 page_size,
368 options,
369 )
370 }
371
372 pub(crate) fn from_complete_metadata(
373 job_service: Arc<JobService>,
374 job_ref: Option<JobReference>,
375 metadata: CompleteQueryMetadata,
376 cached_data: CachedData,
377 page_size: Option<u32>,
378 options: RequestOptions,
379 ) -> Self {
380 let page_token = if metadata.page_token.is_empty() {
381 None
382 } else {
383 Some(metadata.page_token.clone())
384 };
385 Self {
386 job_service,
387 job_ref,
388 cached_data,
389 page_token,
390 metadata,
391 page_size,
392 options,
393 }
394 }
395
396 pub fn read(self) -> RowIterator {
417 RowIterator::new(self)
418 }
419
420 pub fn metadata(&self) -> &CompleteQueryMetadata {
443 &self.metadata
444 }
445
446 pub fn get_job(&self) -> Option<GetJob> {
483 build_get_job(&self.job_service, self.job_ref.as_ref()?, &self.options)
484 }
485}
486
487pub(crate) fn build_get_job(
491 job_service: &JobService,
492 job_ref: &JobReference,
493 options: &RequestOptions,
494) -> Option<GetJob> {
495 if job_ref.job_id.is_empty() {
496 return None;
497 }
498
499 let req = job_service
500 .get_job()
501 .with_options(options.clone())
502 .set_job_id(job_ref.job_id.clone())
503 .set_project_id(job_ref.project_id.clone());
504
505 let req = job_ref
506 .location
507 .clone()
508 .into_iter()
509 .fold(req, |req, location| req.set_location(location));
510
511 Some(req)
512}
513
514pub(crate) async fn poll_query_results(
516 job_service: &JobService,
517 job_ref: &JobReference,
518 backoff_policy: Arc<dyn PollingBackoffPolicy>,
519 options: &RequestOptions,
520) -> Result<GetQueryResultsResponse> {
521 let mut state = PollingState::default();
522
523 loop {
524 let mut req = GetQueryResultsRequest::new()
525 .set_max_results(0u32)
526 .set_project_id(job_ref.project_id.clone())
527 .set_job_id(job_ref.job_id.clone());
528 if let Some(location) = job_ref.location.clone() {
529 req = req.set_location(location);
530 }
531
532 let res = job_service
533 .get_query_results()
534 .with_options(options.clone())
535 .with_request(req)
536 .send()
537 .await?;
538
539 if !res.errors.is_empty() {
540 return Err(QueryError::JobFailed { errors: res.errors });
541 }
542
543 let completed = res.job_complete.unwrap_or(false);
544 if completed {
545 return Ok(res);
546 }
547
548 let delay = backoff_policy.wait_period(&state);
549 tokio::time::sleep(delay).await;
550 state.attempt_count += 1;
552 }
553}
554
555fn build_query_metadata_from_job(resp: Job) -> QueryMetadata {
561 let mut metadata = QueryMetadata::from(resp);
562
563 let query_stats = metadata.statistics.as_ref().and_then(|s| s.query.as_ref());
564
565 metadata.job_complete = metadata.status.as_ref().map(|s| s.state == "DONE");
566 metadata.errors = metadata
567 .status
568 .as_ref()
569 .map(|s| s.errors.clone())
570 .unwrap_or_default();
571 metadata.schema = query_stats.and_then(|q| q.schema.clone());
572 metadata.total_bytes_processed =
573 query_stats
574 .and_then(|q| q.total_bytes_processed)
575 .or_else(|| {
576 metadata
577 .statistics
578 .as_ref()
579 .and_then(|s| s.total_bytes_processed)
580 });
581 metadata.total_bytes_billed = query_stats.and_then(|q| q.total_bytes_billed);
582 metadata.total_slot_ms = query_stats
583 .and_then(|q| q.total_slot_ms)
584 .or_else(|| metadata.statistics.as_ref().and_then(|s| s.total_slot_ms));
585 metadata.cache_hit = query_stats.and_then(|q| q.cache_hit);
586 metadata.num_dml_affected_rows = query_stats.and_then(|q| q.num_dml_affected_rows);
587 metadata.dml_stats = query_stats.and_then(|q| q.dml_stats.clone());
588 metadata.statement_type = query_stats
589 .map(|q| q.statement_type.clone())
590 .unwrap_or_default();
591 metadata.session_info = metadata
592 .statistics
593 .as_ref()
594 .and_then(|s| s.session_info.clone());
595
596 metadata.creation_time = metadata
597 .statistics
598 .as_ref()
599 .and_then(|s| (s.creation_time > 0).then_some(s.creation_time));
600 metadata.start_time = metadata
601 .statistics
602 .as_ref()
603 .and_then(|s| (s.start_time > 0).then_some(s.start_time));
604 metadata.end_time = metadata
605 .statistics
606 .as_ref()
607 .and_then(|s| (s.end_time > 0).then_some(s.end_time));
608
609 if metadata.location.is_empty() {
610 metadata.location = metadata
611 .job_reference
612 .as_ref()
613 .and_then(|r| r.location.clone())
614 .unwrap_or_default();
615 }
616
617 metadata
618}
619
620fn build_complete_query_metadata_from_get_query_results(
623 initial_metadata: QueryMetadata,
624 res: GetQueryResultsResponse,
625 job_ref: &JobReference,
626) -> CompleteQueryMetadata {
627 let mut metadata = CompleteQueryMetadata::from(initial_metadata);
628 if res.schema.is_some() {
629 metadata.schema = res.schema;
630 }
631 if res.total_rows.is_some() {
632 metadata.total_rows = res.total_rows;
633 }
634 if !res.page_token.is_empty() {
635 metadata.page_token = res.page_token;
636 }
637 if res.total_bytes_processed.is_some() {
638 metadata.total_bytes_processed = res.total_bytes_processed;
639 }
640 if res.job_complete.is_some() {
641 metadata.job_complete = res.job_complete;
642 }
643 if !res.errors.is_empty() {
644 metadata.errors = res.errors;
645 }
646 if res.cache_hit.is_some() {
647 metadata.cache_hit = res.cache_hit;
648 }
649 if res.num_dml_affected_rows.is_some() {
650 metadata.num_dml_affected_rows = res.num_dml_affected_rows;
651 }
652 if !res.etag.is_empty() {
653 metadata.etag = res.etag;
654 }
655 if res.job_reference.is_some() {
656 metadata.job_reference = res.job_reference;
657 }
658 if metadata.location.is_empty()
659 && let Some(loc) = job_ref.location.clone()
660 {
661 metadata.location = loc;
662 }
663
664 metadata
665}
666
667#[cfg(test)]
668mod tests {
669 use super::*;
670 use crate::query::builder::{QUERY_REQUEST_ID_PREFIX, Query as QueryBuilder};
671 use crate::query::retry_policy::RetryableJobErrors;
672 use crate::query::tests::{MockJobService, create_job_service, create_test_backoff_policy};
673 use google_cloud_bigquery_v2::model::{
674 ArrowRecordBatch, ArrowSchema, ErrorProto, GetQueryResultsResponse, Job, JobConfiguration,
675 JobReference, QueryResponse, TableFieldSchema, TableSchema,
676 };
677 use google_cloud_gax::error::Error as GaxError;
678 use google_cloud_gax::error::rpc::{Code, Status};
679 use google_cloud_gax::response::Response;
680 use std::time::Duration;
681
682 type TestResult = anyhow::Result<()>;
683
684 impl CompleteQuery {
685 pub(crate) fn from_query_response(
686 job_service: Arc<JobService>,
687 mut query_res: QueryResponse,
688 page_size: Option<u32>,
689 ) -> Self {
690 let cached_data = CachedData::Rows(VecDeque::from(std::mem::take(&mut query_res.rows)));
691 let metadata = QueryMetadata::from(query_res);
692 Self::from_query_metadata(
693 job_service,
694 metadata,
695 cached_data,
696 page_size,
697 RequestOptions::default(),
698 )
699 }
700 }
701
702 #[tokio::test]
703 async fn test_query_until_done_already_completed() -> TestResult {
704 let job_service = create_job_service(MockJobService::new());
705 let job_ref = JobReference::new()
706 .set_project_id("some_project")
707 .set_job_id("some_job_id");
708 let query_res = QueryResponse::new()
709 .set_job_complete(true)
710 .set_job_reference(job_ref.clone())
711 .set_schema(TableSchema::new())
712 .set_page_token("some_page_token")
713 .set_rows([wkt::Struct::new()])
714 .set_cache_hit(true);
715
716 let query = Query::from_query_response(
717 job_service,
718 query_res,
719 None,
720 None,
721 RequestOptions::default(),
722 );
723
724 let completed = query.until_done().await?;
725 assert_eq!(completed.job_ref.as_ref().unwrap().job_id, "some_job_id");
726 assert_eq!(completed.page_token, Some("some_page_token".to_string()));
727 match &completed.cached_data {
728 CachedData::Rows(rows) => assert_eq!(rows.len(), 1),
729 _ => panic!("expected rows"),
730 }
731
732 let metadata = completed.metadata();
733 assert_eq!(metadata.cache_hit, Some(true));
734 assert_eq!(metadata.job_complete, Some(true));
735 assert_eq!(metadata.page_token, "some_page_token".to_string());
736
737 Ok(())
738 }
739
740 #[tokio::test]
741 async fn test_query_until_done_already_completed_arrow() -> TestResult {
742 let job_service = create_job_service(MockJobService::new());
743 let job_ref = JobReference::new()
744 .set_project_id("some_project")
745 .set_job_id("some_job_id");
746 let query_res = QueryResponse::new()
747 .set_job_complete(true)
748 .set_job_reference(job_ref.clone())
749 .set_arrow_schema(
750 ArrowSchema::new().set_serialized_schema(Bytes::from_static(b"test_schema")),
751 )
752 .set_arrow_record_batch(
753 ArrowRecordBatch::new()
754 .set_serialized_record_batch(Bytes::from_static(b"test_batch")),
755 )
756 .set_cache_hit(true);
757
758 let query = Query::from_query_response(
759 job_service,
760 query_res,
761 None,
762 None,
763 RequestOptions::default(),
764 );
765
766 let completed = query.until_done().await?;
767 assert_eq!(completed.job_ref.as_ref().unwrap().job_id, "some_job_id");
768 match &completed.cached_data {
769 CachedData::Arrow {
770 serialized_record_batch,
771 serialized_schema,
772 } => {
773 assert_eq!(serialized_record_batch, &Bytes::from_static(b"test_batch"));
774 assert_eq!(serialized_schema, &Bytes::from_static(b"test_schema"));
775 }
776 _ => panic!("expected arrow cached data"),
777 }
778
779 Ok(())
780 }
781
782 #[tokio::test]
783 async fn test_query_until_done_preserves_page_size() -> TestResult {
784 let job_service = create_job_service(MockJobService::new());
785 let job_ref = JobReference::new()
786 .set_project_id("some_project")
787 .set_job_id("some_job_id");
788 let query_res = QueryResponse::new()
789 .set_job_complete(true)
790 .set_job_reference(job_ref.clone())
791 .set_schema(TableSchema::new());
792
793 let query = Query::from_query_response(
794 job_service,
795 query_res,
796 None,
797 Some(42),
798 RequestOptions::default(),
799 );
800
801 let completed = query.until_done().await?;
802 assert_eq!(completed.page_size, Some(42));
803
804 Ok(())
805 }
806
807 #[tokio::test]
808 async fn test_query_until_done_polls_success() -> TestResult {
809 let mut mock = MockJobService::new();
810 mock.expect_get_query_results()
811 .returning(|req, _| {
812 assert_eq!(req.project_id, "some_project");
813 assert_eq!(req.job_id, "some_job_id");
814 assert_eq!(req.max_results, Some(0));
815 assert_eq!(req.location, "us-central1");
816 let res = GetQueryResultsResponse::new()
817 .set_job_complete(true)
818 .set_job_reference(JobReference::new().set_job_id(req.job_id))
819 .set_schema(TableSchema::new())
820 .set_page_token("some_page_token")
821 .set_rows(vec![wkt::Struct::new(), wkt::Struct::new()])
822 .set_cache_hit(false);
823 Ok(Response::from(res))
824 })
825 .times(1);
826 let job_service = create_job_service(mock);
827 let job_ref = JobReference::new()
828 .set_project_id("some_project")
829 .set_job_id("some_job_id")
830 .set_location("us-central1");
831 let query_res = QueryResponse::new()
832 .set_job_complete(false)
833 .set_job_reference(job_ref);
834
835 let query = Query::from_query_response(
836 job_service,
837 query_res,
838 None,
839 None,
840 RequestOptions::default(),
841 );
842
843 let completed = query.until_done().await?;
844 assert_eq!(completed.job_ref.as_ref().unwrap().job_id, "some_job_id");
845 assert_eq!(completed.page_token, Some("some_page_token".to_string()));
846 match &completed.cached_data {
847 CachedData::Rows(rows) => assert_eq!(rows.len(), 2),
848 _ => panic!("expected rows"),
849 }
850
851 let metadata = completed.metadata();
852 assert_eq!(metadata.cache_hit, Some(false));
853 assert_eq!(metadata.job_complete, Some(true));
854 assert_eq!(metadata.page_token, "some_page_token");
855
856 Ok(())
857 }
858
859 #[tokio::test(start_paused = true)]
860 async fn test_poll_query_results_loops_until_complete() -> TestResult {
861 let mut mock = MockJobService::new();
862 let mut backoff_policy = create_test_backoff_policy();
863 backoff_policy
864 .expect_wait_period()
865 .times(2)
866 .return_const(Duration::from_millis(1));
867
868 let mut seq = mockall::Sequence::new();
869
870 mock.expect_get_query_results()
871 .in_sequence(&mut seq)
872 .times(2)
873 .returning(|_, _| {
874 Ok(Response::from(
875 GetQueryResultsResponse::new().set_job_complete(false),
876 ))
877 });
878
879 mock.expect_get_query_results()
880 .in_sequence(&mut seq)
881 .times(1)
882 .returning(|_, _| {
883 Ok(Response::from(
884 GetQueryResultsResponse::new().set_job_complete(true),
885 ))
886 });
887
888 let job_service = create_job_service(mock);
889 let job_ref = JobReference::new()
890 .set_project_id("some_project")
891 .set_job_id("some_job_id");
892
893 let res = poll_query_results(
894 &job_service,
895 &job_ref,
896 Arc::new(backoff_policy),
897 &RequestOptions::default(),
898 )
899 .await?;
900
901 assert!(res.job_complete.unwrap(), "{res:?}");
902
903 Ok(())
904 }
905
906 #[tokio::test]
907 async fn test_query_until_done_job_failed_error() -> TestResult {
908 let mut mock = MockJobService::new();
909 mock.expect_get_query_results().returning(|req, _| {
910 assert_eq!(req.project_id, "some_project");
911 assert_eq!(req.job_id, "some_job_id");
912 assert_eq!(req.max_results, Some(0));
913 let err_proto = ErrorProto::new()
914 .set_reason("invalidQuery")
915 .set_message("Syntax error");
916 let res = GetQueryResultsResponse::new().set_errors(vec![err_proto]);
917 Ok(Response::from(res))
918 });
919 let job_service = create_job_service(mock);
920 let job_ref = JobReference::new()
921 .set_project_id("some_project")
922 .set_job_id("some_job_id");
923 let query_res = QueryResponse::new()
924 .set_job_complete(false)
925 .set_job_reference(job_ref);
926
927 let query = Query::from_query_response(
928 job_service,
929 query_res,
930 None,
931 None,
932 RequestOptions::default(),
933 );
934
935 let err = query.until_done().await.unwrap_err();
936 let errors = match err {
937 QueryError::JobFailed { errors } => errors,
938 _ => panic!("expected QueryError::JobFailed, got {err:?}"),
939 };
940 assert_eq!(
941 errors,
942 [ErrorProto::new()
943 .set_reason("invalidQuery")
944 .set_message("Syntax error")]
945 );
946
947 Ok(())
948 }
949
950 #[tokio::test]
951 async fn test_query_until_done_rpc_error() -> TestResult {
952 let mut mock = MockJobService::new();
953 mock.expect_get_query_results().returning(|req, _| {
954 assert_eq!(req.project_id, "some_project");
955 assert_eq!(req.job_id, "some_job_id");
956 assert_eq!(req.max_results, Some(0));
957 let status = Status::default()
958 .set_code(Code::InvalidArgument)
959 .set_message("simulated bad request");
960 Err(GaxError::service(status))
961 });
962 let job_service = create_job_service(mock);
963 let job_ref = JobReference::new()
964 .set_project_id("some_project")
965 .set_job_id("some_job_id");
966 let query_res = QueryResponse::new()
967 .set_job_complete(false)
968 .set_job_reference(job_ref);
969
970 let query = Query::from_query_response(
971 job_service,
972 query_res,
973 None,
974 None,
975 RequestOptions::default(),
976 );
977
978 let err = query.until_done().await.unwrap_err();
979 let source = match err {
980 QueryError::Rpc { source } => source,
981 _ => panic!("expected QueryError::Rpc, got {err:?}"),
982 };
983 assert_eq!(source.status().unwrap().code, Code::InvalidArgument);
984
985 Ok(())
986 }
987
988 #[tokio::test]
989 async fn test_complete_query_read() -> TestResult {
990 let job_service = create_job_service(MockJobService::new());
991 let job_ref = JobReference::new()
992 .set_project_id("some_project")
993 .set_job_id("some_job_id");
994 let schema = TableSchema::new().set_fields([TableFieldSchema::new()
995 .set_name("name")
996 .set_type("STRING")
997 .set_mode("NULLABLE")]);
998 let row = serde_json::Map::from_iter([(
999 "f".to_string(),
1000 serde_json::json!([{ "v": "test_name" }]),
1001 )]);
1002 let query_res = QueryResponse::new()
1003 .set_job_complete(true)
1004 .set_job_reference(job_ref)
1005 .set_schema(schema)
1006 .set_rows(vec![row]);
1007
1008 let complete_query = CompleteQuery::from_query_response(job_service, query_res, None);
1009
1010 let mut iter = complete_query.read();
1011 let row = iter.next().await.expect("should return first row")?;
1012 assert_eq!(row.get::<String>("name")?, "test_name");
1013 assert!(iter.next().await.is_none(), "{iter:?}");
1014
1015 Ok(())
1016 }
1017
1018 #[tokio::test(start_paused = true)]
1019 async fn test_query_until_done_reissue_on_retryable_job_failed() -> TestResult {
1020 let mut mock = MockJobService::new();
1021 let mut seq = mockall::Sequence::new();
1022
1023 mock.expect_get_query_results()
1024 .in_sequence(&mut seq)
1025 .times(1)
1026 .returning(|req, options| {
1027 assert_eq!(options.quota_project().as_deref(), Some("some_project"));
1028 assert_eq!(req.job_id, "initial_job_id");
1029 let err_proto = ErrorProto::new()
1030 .set_reason("backendError")
1031 .set_message("temporary server issue");
1032 let res = GetQueryResultsResponse::new().set_errors(vec![err_proto]);
1033 Ok(Response::from(res))
1034 });
1035
1036 mock.expect_query()
1037 .in_sequence(&mut seq)
1038 .times(1)
1039 .returning(|req, options| {
1040 assert_eq!(options.quota_project().as_deref(), Some("some_project"));
1041 let req_id = &req.query_request.as_ref().unwrap().request_id;
1042 assert!(req_id.starts_with(QUERY_REQUEST_ID_PREFIX));
1043 assert!(uuid::Uuid::parse_str(&req_id[QUERY_REQUEST_ID_PREFIX.len()..]).is_ok());
1044 let new_job_ref = JobReference::new()
1045 .set_project_id("some_project")
1046 .set_job_id("reissued_job_id");
1047 Ok(Response::from(
1048 QueryResponse::new()
1049 .set_job_complete(false)
1050 .set_job_reference(new_job_ref)
1051 .set_schema(TableSchema::new()),
1052 ))
1053 });
1054
1055 mock.expect_get_query_results()
1056 .in_sequence(&mut seq)
1057 .times(1)
1058 .returning(|req, options| {
1059 assert_eq!(options.quota_project().as_deref(), Some("some_project"));
1060 assert_eq!(req.job_id, "reissued_job_id");
1061 let res = GetQueryResultsResponse::new()
1062 .set_job_complete(true)
1063 .set_schema(TableSchema::new());
1064 Ok(Response::from(res))
1065 });
1066
1067 let job_service = create_job_service(mock);
1068 let job_ref = JobReference::new()
1069 .set_project_id("some_project")
1070 .set_job_id("initial_job_id");
1071 let query_res = QueryResponse::new()
1072 .set_job_complete(false)
1073 .set_job_reference(job_ref);
1074
1075 let query_builder = QueryBuilder::new(job_service.clone(), "SELECT 1".to_string())
1076 .with_project_id("some_project");
1077 let options = query_builder.options.clone();
1078 let mut retry_context = RetryContext::new(query_builder);
1079 retry_context.state.attempt_count = 1;
1080
1081 let query =
1082 Query::from_query_response(job_service, query_res, Some(&retry_context), None, options);
1083
1084 let completed = query.until_done().await?;
1085 assert_eq!(
1086 completed.job_ref.as_ref().unwrap().job_id,
1087 "reissued_job_id"
1088 );
1089 assert_eq!(
1090 completed.options.quota_project().as_deref(),
1091 Some("some_project")
1092 );
1093 Ok(())
1094 }
1095
1096 #[tokio::test(start_paused = true)]
1097 async fn test_query_until_done_reissue_retry_exhausted() -> TestResult {
1098 let mut mock = MockJobService::new();
1099 let mut seq = mockall::Sequence::new();
1100
1101 mock.expect_get_query_results()
1103 .in_sequence(&mut seq)
1104 .times(1)
1105 .returning(|req, _| {
1106 assert_eq!(req.job_id, "initial_job_id");
1107 let err_proto = ErrorProto::new()
1108 .set_reason("backendError")
1109 .set_message("first temporary server issue");
1110 let res = GetQueryResultsResponse::new().set_errors(vec![err_proto]);
1111 Ok(Response::from(res))
1112 });
1113
1114 mock.expect_query()
1116 .in_sequence(&mut seq)
1117 .times(1)
1118 .returning(|req, _| {
1119 let req_id = &req.query_request.as_ref().unwrap().request_id;
1120 assert!(req_id.starts_with(QUERY_REQUEST_ID_PREFIX));
1121 assert!(uuid::Uuid::parse_str(&req_id[QUERY_REQUEST_ID_PREFIX.len()..]).is_ok());
1122 let new_job_ref = JobReference::new()
1123 .set_project_id("some_project")
1124 .set_job_id("reissued_job_id");
1125 Ok(Response::from(
1126 QueryResponse::new()
1127 .set_job_complete(false)
1128 .set_job_reference(new_job_ref)
1129 .set_schema(TableSchema::new()),
1130 ))
1131 });
1132
1133 mock.expect_get_query_results()
1135 .in_sequence(&mut seq)
1136 .times(1)
1137 .returning(|req, _| {
1138 assert_eq!(req.job_id, "reissued_job_id");
1139 let err_proto = ErrorProto::new()
1140 .set_reason("backendError")
1141 .set_message("second temporary server issue");
1142 let res = GetQueryResultsResponse::new().set_errors(vec![err_proto]);
1143 Ok(Response::from(res))
1144 });
1145
1146 let job_service = create_job_service(mock);
1147 let job_ref = JobReference::new()
1148 .set_project_id("some_project")
1149 .set_job_id("initial_job_id");
1150
1151 let query_res = QueryResponse::new()
1152 .set_job_complete(false)
1153 .set_job_reference(job_ref)
1154 .set_schema(TableSchema::new());
1155 let mut query_builder = QueryBuilder::new(job_service.clone(), "SELECT 1".to_string())
1156 .with_project_id("some_project");
1157 query_builder.job_retry_policy =
1158 Arc::new(RetryableJobErrors::default().with_attempt_limit(2));
1159 let mut retry_context = RetryContext::new(query_builder);
1160 retry_context.state.attempt_count = 1;
1161
1162 let query = Query::from_query_response(
1163 job_service,
1164 query_res,
1165 Some(&retry_context),
1166 None,
1167 RequestOptions::default(),
1168 );
1169
1170 let err = query.until_done().await.unwrap_err();
1171 let errors = match err {
1172 QueryError::JobFailed { errors } => errors,
1173 _ => panic!("expected QueryError::JobFailed, got {err:?}"),
1174 };
1175 assert_eq!(errors.len(), 1);
1176 assert_eq!(errors[0].reason, "backendError");
1177 assert_eq!(errors[0].message, "second temporary server issue");
1178
1179 Ok(())
1180 }
1181
1182 #[tokio::test]
1183 async fn test_query_get_job_success() -> TestResult {
1184 let mut mock = MockJobService::new();
1185 mock.expect_get_job().returning(|req, _| {
1186 assert_eq!(req.project_id, "some_project");
1187 assert_eq!(req.job_id, "some_job_id");
1188 assert_eq!(req.location, "us-central1");
1189 let res = Job::new()
1190 .set_job_reference(JobReference::new().set_job_id(req.job_id))
1191 .set_user_email("test@example.com");
1192 Ok(Response::from(res))
1193 });
1194 let job_service = create_job_service(mock);
1195 let job_ref = JobReference::new()
1196 .set_project_id("some_project")
1197 .set_job_id("some_job_id")
1198 .set_location("us-central1");
1199 let query_res = QueryResponse::new()
1200 .set_schema(TableSchema::new())
1201 .set_job_reference(job_ref);
1202
1203 let query = Query::from_query_response(
1204 job_service.clone(),
1205 query_res.clone(),
1206 None,
1207 None,
1208 RequestOptions::default(),
1209 );
1210 let job = query.get_job().unwrap().send().await?;
1211 assert_eq!(job.user_email, "test@example.com");
1212
1213 let complete_query = CompleteQuery::from_query_response(job_service, query_res, None);
1214 let job = complete_query.get_job().unwrap().send().await?;
1215 assert_eq!(job.user_email, "test@example.com");
1216 Ok(())
1217 }
1218
1219 #[tokio::test]
1220 async fn test_query_get_job_empty_job_id() -> TestResult {
1221 let job_service = create_job_service(MockJobService::new());
1222 let job_ref = JobReference::new()
1224 .set_location("us-central1")
1225 .set_project_id("some-project");
1226 let query_res = QueryResponse::new()
1227 .set_schema(TableSchema::new())
1228 .set_job_reference(job_ref);
1229
1230 let query = Query::from_query_response(
1231 job_service.clone(),
1232 query_res.clone(),
1233 None,
1234 None,
1235 RequestOptions::default(),
1236 );
1237 assert!(query.get_job().is_none(), "{query:?}");
1238
1239 let complete_query = CompleteQuery::from_query_response(job_service, query_res, None);
1240 assert!(complete_query.get_job().is_none(), "{complete_query:?}");
1241 Ok(())
1242 }
1243
1244 #[tokio::test]
1245 async fn test_query_get_job_rpc_error() -> TestResult {
1246 let mut mock = MockJobService::new();
1247 mock.expect_get_job().returning(|req, _| {
1248 assert_eq!(req.project_id, "some_project");
1249 assert_eq!(req.job_id, "some_job_id");
1250 let status = Status::default()
1251 .set_code(Code::NotFound)
1252 .set_message("job not found");
1253 Err(GaxError::service(status))
1254 });
1255 let job_service = create_job_service(mock);
1256 let job_ref = JobReference::new()
1257 .set_project_id("some_project")
1258 .set_job_id("some_job_id");
1259 let query_res = QueryResponse::new()
1260 .set_schema(TableSchema::new())
1261 .set_job_reference(job_ref);
1262
1263 let query = Query::from_query_response(
1264 job_service.clone(),
1265 query_res.clone(),
1266 None,
1267 None,
1268 RequestOptions::default(),
1269 );
1270 let req = query.get_job().unwrap();
1271 let err = req.send().await.unwrap_err();
1272 assert_eq!(err.status().unwrap().code, Code::NotFound);
1273
1274 let complete_query = CompleteQuery::from_query_response(job_service, query_res, None);
1275 let req = complete_query.get_job().unwrap();
1276 let err = req.send().await.unwrap_err();
1277 assert_eq!(err.status().unwrap().code, Code::NotFound);
1278 Ok(())
1279 }
1280
1281 #[tokio::test]
1282 async fn test_query_until_done_dry_run_job_returns_error() -> TestResult {
1283 let job_service = create_job_service(MockJobService::new());
1284 let job_ref = JobReference::new()
1285 .set_project_id("some_project")
1286 .set_location("US");
1287 let job = Job::new()
1288 .set_job_reference(job_ref)
1289 .set_configuration(JobConfiguration::new().set_dry_run(true));
1290
1291 let query = Query::from_job(job_service, job, None, None, RequestOptions::default());
1292 let err = query.until_done().await.unwrap_err();
1293 assert!(
1294 matches!(err, QueryError::DryRun),
1295 "expected DryRun error, got {err:?}"
1296 );
1297 Ok(())
1298 }
1299
1300 #[tokio::test(start_paused = true)]
1301 async fn test_query_until_done_initial_poll_delay() -> TestResult {
1302 let mut mock = MockJobService::new();
1303 let mut seq = mockall::Sequence::new();
1304 mock.expect_get_query_results()
1305 .times(1)
1306 .in_sequence(&mut seq)
1307 .returning(|_, _| {
1308 Ok(Response::from(
1309 GetQueryResultsResponse::new().set_job_complete(false),
1310 ))
1311 });
1312 mock.expect_get_query_results()
1313 .times(1)
1314 .in_sequence(&mut seq)
1315 .returning(|_, _| {
1316 Ok(Response::from(
1317 GetQueryResultsResponse::new().set_job_complete(true),
1318 ))
1319 });
1320
1321 let job_service = create_job_service(mock);
1322 let job_ref = JobReference::new()
1323 .set_project_id("some_project")
1324 .set_job_id("some_job_id");
1325 let query_res = QueryResponse::new()
1326 .set_job_complete(false)
1327 .set_job_reference(job_ref);
1328
1329 let start = tokio::time::Instant::now();
1330 let query = Query::from_query_response(
1331 job_service,
1332 query_res,
1333 None,
1334 None,
1335 RequestOptions::default(),
1336 );
1337 let _completed = query.until_done().await?;
1338 let elapsed = start.elapsed();
1339
1340 assert!(
1341 elapsed <= Duration::from_secs(1),
1342 "expected initial poll delay <= 1s, got {elapsed:?}"
1343 );
1344 Ok(())
1345 }
1346
1347 #[test]
1348 fn test_build_query_metadata_from_job() {
1349 use google_cloud_bigquery_v2::model::{
1350 JobConfigurationQuery, JobCreationReason, JobStatistics, JobStatistics2, JobStatus,
1351 };
1352
1353 let job =
1354 Job::new()
1355 .set_id("rust-sdk-testing:US.job_123")
1356 .set_kind("bigquery#job")
1357 .set_etag("etag123")
1358 .set_self_link("https://www.googleapis.com/bigquery/v2/...")
1359 .set_user_email("user@example.com")
1360 .set_principal_subject("user:user@example.com")
1361 .set_job_creation_reason(JobCreationReason::new().set_code(
1362 google_cloud_bigquery_v2::model::job_creation_reason::Code::Requested,
1363 ))
1364 .set_job_reference(
1365 JobReference::new()
1366 .set_project_id("rust-sdk-testing")
1367 .set_job_id("job_123")
1368 .set_location("US"),
1369 )
1370 .set_configuration(
1371 JobConfiguration::new().set_query(
1372 JobConfigurationQuery::new()
1373 .set_query("SELECT 1 AS one")
1374 .set_use_legacy_sql(false),
1375 ),
1376 )
1377 .set_status(JobStatus::new().set_state("DONE"))
1378 .set_statistics(
1379 JobStatistics::new()
1380 .set_creation_time(1790020718574i64)
1381 .set_start_time(1790020718592i64)
1382 .set_end_time(1790020718847i64)
1383 .set_total_bytes_processed(0i64)
1384 .set_query(
1385 JobStatistics2::new()
1386 .set_total_bytes_processed(0i64)
1387 .set_total_bytes_billed(0i64)
1388 .set_cache_hit(true)
1389 .set_statement_type("SELECT")
1390 .set_schema(TableSchema::new().set_fields([
1391 TableFieldSchema::new().set_name("one").set_type("INTEGER"),
1392 ])),
1393 ),
1394 );
1395
1396 let metadata = build_query_metadata_from_job(job);
1397
1398 assert_eq!(metadata.id, "rust-sdk-testing:US.job_123");
1399 assert_eq!(metadata.job_complete, Some(true));
1400 assert_eq!(metadata.creation_time, Some(1790020718574));
1401 assert_eq!(metadata.start_time, Some(1790020718592));
1402 assert_eq!(metadata.end_time, Some(1790020718847));
1403 assert_eq!(metadata.cache_hit, Some(true));
1404 assert_eq!(metadata.statement_type, "SELECT");
1405 assert_eq!(metadata.location, "US");
1406 assert_eq!(metadata.total_bytes_processed, Some(0));
1407 assert_eq!(metadata.total_bytes_billed, Some(0));
1408 assert!(metadata.schema.is_some());
1409 }
1410
1411 #[tokio::test]
1412 async fn test_query_until_done_preserves_metadata_fields_from_job() -> TestResult {
1413 use google_cloud_bigquery_v2::model::{
1414 JobConfigurationQuery, JobCreationReason, JobStatistics, JobStatistics2, JobStatus,
1415 };
1416
1417 let mut mock = MockJobService::new();
1418 mock.expect_get_query_results()
1419 .returning(|req, _| {
1420 let res = GetQueryResultsResponse::new()
1421 .set_job_complete(true)
1422 .set_job_reference(JobReference::new().set_job_id(req.job_id))
1423 .set_schema(TableSchema::new())
1424 .set_rows(vec![wkt::Struct::new()])
1425 .set_cache_hit(true);
1426 Ok(Response::from(res))
1427 })
1428 .times(1);
1429
1430 let job_service = create_job_service(mock);
1431 let job =
1432 Job::new()
1433 .set_id("rust-sdk-testing:US.job_123")
1434 .set_kind("bigquery#job")
1435 .set_job_creation_reason(JobCreationReason::new().set_code(
1436 google_cloud_bigquery_v2::model::job_creation_reason::Code::Requested,
1437 ))
1438 .set_job_reference(
1439 JobReference::new()
1440 .set_project_id("rust-sdk-testing")
1441 .set_job_id("job_123")
1442 .set_location("US"),
1443 )
1444 .set_configuration(
1445 JobConfiguration::new().set_query(
1446 JobConfigurationQuery::new()
1447 .set_query("SELECT 1 AS one")
1448 .set_use_legacy_sql(false),
1449 ),
1450 )
1451 .set_status(JobStatus::new().set_state("DONE"))
1452 .set_statistics(
1453 JobStatistics::new()
1454 .set_creation_time(1790020718574i64)
1455 .set_start_time(1790020718592i64)
1456 .set_end_time(1790020718847i64)
1457 .set_total_bytes_processed(0i64)
1458 .set_query(
1459 JobStatistics2::new()
1460 .set_total_bytes_processed(0i64)
1461 .set_total_bytes_billed(0i64)
1462 .set_cache_hit(true)
1463 .set_statement_type("SELECT"),
1464 ),
1465 );
1466
1467 let query = Query::from_job(job_service, job, None, None, RequestOptions::default());
1468 let complete = query.until_done().await?;
1469 let meta = complete.metadata();
1470
1471 assert_eq!(meta.creation_time, Some(1790020718574));
1472 assert_eq!(meta.start_time, Some(1790020718592));
1473 assert_eq!(meta.end_time, Some(1790020718847));
1474 assert_eq!(meta.statement_type, "SELECT");
1475 assert_eq!(meta.location, "US");
1476 assert_eq!(meta.cache_hit, Some(true));
1477
1478 Ok(())
1479 }
1480
1481 #[tokio::test]
1482 async fn test_query_until_done_polls_with_user_agent() -> TestResult {
1483 let mut mock = MockJobService::new();
1484 mock.expect_get_query_results()
1485 .returning(|req, options| {
1486 assert_eq!(options.user_agent().as_deref(), Some("my-app/1.2.3"));
1487 let res = GetQueryResultsResponse::new()
1488 .set_job_complete(true)
1489 .set_job_reference(JobReference::new().set_job_id(req.job_id))
1490 .set_schema(TableSchema::new());
1491 Ok(Response::from(res))
1492 })
1493 .times(1);
1494 let job_service = create_job_service(mock);
1495 let job_ref = JobReference::new()
1496 .set_project_id("some_project")
1497 .set_job_id("some_job_id");
1498 let query_res = QueryResponse::new()
1499 .set_job_complete(false)
1500 .set_job_reference(job_ref);
1501
1502 let mut options = RequestOptions::default();
1503 options.set_user_agent("my-app/1.2.3");
1504 let query = Query::from_query_response(job_service, query_res, None, None, options);
1505
1506 let completed = query.until_done().await?;
1507 assert_eq!(
1508 completed.options.user_agent().as_deref(),
1509 Some("my-app/1.2.3")
1510 );
1511
1512 Ok(())
1513 }
1514
1515 #[tokio::test]
1516 async fn test_query_get_job_with_user_agent() -> TestResult {
1517 let mut mock = MockJobService::new();
1518 mock.expect_get_job().times(2).returning(|req, options| {
1519 assert_eq!(options.user_agent().as_deref(), Some("my-app/1.2.3"));
1520 let res = Job::new()
1521 .set_job_reference(JobReference::new().set_job_id(req.job_id))
1522 .set_user_email("test@example.com");
1523 Ok(Response::from(res))
1524 });
1525 let job_service = create_job_service(mock);
1526 let job_ref = JobReference::new()
1527 .set_project_id("some_project")
1528 .set_job_id("some_job_id");
1529 let query_res = QueryResponse::new()
1530 .set_job_complete(true)
1531 .set_schema(TableSchema::new())
1532 .set_job_reference(job_ref);
1533
1534 let mut options = RequestOptions::default();
1535 options.set_user_agent("my-app/1.2.3");
1536 let query = Query::from_query_response(job_service, query_res, None, None, options);
1537 let job = query.get_job().unwrap().send().await?;
1538 assert_eq!(job.user_email, "test@example.com");
1539
1540 let complete_query = query.until_done().await?;
1541 let job = complete_query.get_job().unwrap().send().await?;
1542 assert_eq!(job.user_email, "test@example.com");
1543 Ok(())
1544 }
1545
1546 #[tokio::test]
1547 async fn test_query_until_done_polls_with_quota_project() -> TestResult {
1548 let mut mock = MockJobService::new();
1549 mock.expect_get_query_results()
1550 .returning(|req, options| {
1551 assert_eq!(options.quota_project().as_deref(), Some("some_project"));
1552 let res = GetQueryResultsResponse::new()
1553 .set_job_complete(true)
1554 .set_job_reference(JobReference::new().set_job_id(req.job_id))
1555 .set_schema(TableSchema::new());
1556 Ok(Response::from(res))
1557 })
1558 .times(1);
1559 let job_service = create_job_service(mock);
1560 let job_ref = JobReference::new()
1561 .set_project_id("some_project")
1562 .set_job_id("some_job_id");
1563 let query_res = QueryResponse::new()
1564 .set_job_complete(false)
1565 .set_job_reference(job_ref);
1566
1567 let mut options = RequestOptions::default();
1568 options.set_quota_project("some_project");
1569 let query = Query::from_query_response(job_service, query_res, None, None, options);
1570
1571 let completed = query.until_done().await?;
1572 assert_eq!(
1573 completed.options.quota_project().as_deref(),
1574 Some("some_project")
1575 );
1576
1577 Ok(())
1578 }
1579
1580 #[tokio::test]
1581 async fn test_query_get_job_with_quota_project() -> TestResult {
1582 let mut mock = MockJobService::new();
1583 mock.expect_get_job().times(2).returning(|req, options| {
1584 assert_eq!(options.quota_project().as_deref(), Some("some_project"));
1585 let res = Job::new()
1586 .set_job_reference(JobReference::new().set_job_id(req.job_id))
1587 .set_user_email("test@example.com");
1588 Ok(Response::from(res))
1589 });
1590 let job_service = create_job_service(mock);
1591 let job_ref = JobReference::new()
1592 .set_project_id("some_project")
1593 .set_job_id("some_job_id");
1594 let query_res = QueryResponse::new()
1595 .set_job_complete(true)
1596 .set_schema(TableSchema::new())
1597 .set_job_reference(job_ref);
1598
1599 let mut options = RequestOptions::default();
1600 options.set_quota_project("some_project");
1601 let query = Query::from_query_response(job_service, query_res, None, None, options);
1602 let job = query.get_job().unwrap().send().await?;
1603 assert_eq!(job.user_email, "test@example.com");
1604
1605 let complete_query = query.until_done().await?;
1606 let job = complete_query.get_job().unwrap().send().await?;
1607 assert_eq!(job.user_email, "test@example.com");
1608 Ok(())
1609 }
1610}