use arrow_array::RecordBatch;
use arrow_odbc::odbc_api::IntoParameter;
use tokio::sync::mpsc::Sender;
use crate::odbc_source::_helpers::batches::{batch_reader, fetch_error};
use crate::odbc_source::_helpers::connect::connect;
use crate::odbc_source::_helpers::convert::{converted, output_schema};
use crate::odbc_source::_helpers::query::select_query;
use crate::odbc_source::constants::BATCHES_AHEAD;
use crate::odbc_source::errors::OdbcSourceError;
use crate::odbc_source::models::{OdbcReadRequest, OdbcReader};
pub async fn read_rows(
reader: &OdbcReader,
request: &OdbcReadRequest<'_>,
mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>,
) -> Result<u64, OdbcSourceError> {
let schema = output_schema(request.columns).map_err(OdbcSourceError::Invalid)?;
let query = select_query(request);
let (sender, mut receiver) = tokio::sync::mpsc::channel(BATCHES_AHEAD);
let owned = (
reader.clone(),
request.batch_rows,
request.parameters.to_vec(),
);
let producer =
tokio::task::spawn_blocking(move || fetch(&owned.0, (&query, &owned.2), owned.1, &sender));
let mut rows: u64 = 0;
while let Some(batch) = receiver.recv().await {
let batch = converted(&batch?, &schema).map_err(OdbcSourceError::Invalid)?;
rows += batch.num_rows() as u64;
on_batch(batch).await.map_err(OdbcSourceError::Consumer)?;
}
producer
.await
.map_err(|error| OdbcSourceError::Invalid(format!("the ODBC read stopped: {error}")))??;
Ok(rows)
}
fn fetch(
reader: &OdbcReader,
(query, parameters): (&str, &[String]),
batch_rows: usize,
sender: &Sender<Result<RecordBatch, OdbcSourceError>>,
) -> Result<(), OdbcSourceError> {
let connection = connect(reader)?;
let dbms: String = connection.database_management_system_name()?;
let parameters: Vec<_> = parameters
.iter()
.map(|value| value.as_str().into_parameter())
.collect();
let Some(cursor) = connection.execute(query, parameters.as_slice(), None)? else {
return Err(OdbcSourceError::Invalid(
"the read returned no result set".to_owned(),
));
};
let batches = batch_reader(reader, cursor, (dbms, batch_rows))?;
for batch in batches {
let batch = batch.map_err(|error| fetch_error(&error.to_string()));
let failed = batch.is_err();
if sender.blocking_send(batch).is_err() || failed {
break;
}
}
Ok(())
}