mod params;
pub use params::*;
mod decoder;
mod ipc;
mod keys;
mod projection;
mod session;
mod stream;
mod support;
pub(crate) use decoder::decode_rows;
pub use decoder::BigQueryBatchRows;
pub(crate) use ipc::ArrowIpcDecoder;
pub(crate) use support::skip_failed_rows;
use crate::{BigQueryDb, BigQueryResult};
use arrow_array::RecordBatch;
use futures::stream::BoxStream;
use serde::de::DeserializeOwned;
use session::Projection;
use tracing::field::Empty;
impl BigQueryReadParams {
fn span(&self) -> tracing::Span {
tracing::debug_span!(
"BigQuery Read",
"/bigquery/table" = %self.table,
"/bigquery/streams" = Empty,
"/bigquery/estimated_bytes_scanned" = Empty,
"/bigquery/estimated_rows" = Empty,
"/bigquery/rows_read" = Empty,
"/bigquery/bytes_read" = Empty,
"/bigquery/throttle_percent" = Empty,
)
}
}
impl BigQueryDb {
pub(crate) async fn read_table_batches<'b>(
&self,
params: BigQueryReadParams,
) -> BigQueryResult<BoxStream<'b, BigQueryResult<RecordBatch>>> {
let span = params.span();
let session = self
.open_read_session(¶ms, Projection::All, &span)
.await?;
session.record(&span);
Ok(self
.start_read_streams(session, &span, |batch, _| batch)
.into_batches())
}
pub(crate) async fn read_table_rows<'b, T>(
&self,
params: BigQueryReadParams,
) -> BigQueryResult<BoxStream<'b, BigQueryResult<T>>>
where
T: DeserializeOwned + Send + 'static,
{
let span = params.span();
let projection = match projection::struct_fields::<T>() {
Some(fields) => Projection::Auto(fields),
None => Projection::All,
};
let session = self.open_read_session(¶ms, projection, &span).await?;
session.record(&span);
Ok(self
.start_read_streams(session, &span, |batch, first_row| {
decode_rows::<T>(&batch, first_row)
})
.into_rows())
}
}
#[cfg(test)]
mod tests;