use arrow_array::RecordBatch;
use tokio_postgres::Client;
use crate::postgres_source::_helpers::copy_stream::{join_reads, start_reads};
use crate::postgres_source::_helpers::split_read::split_readers;
use crate::postgres_source::constants::COPY_QUEUE_BATCHES;
use crate::postgres_source::errors::PostgresError;
use crate::postgres_source::main::end_snapshot::end_snapshot;
use crate::postgres_source::models::CopyRequest;
pub async fn copy_table(
client: &Client,
request: CopyRequest<'_>,
mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>,
) -> Result<u64, PostgresError> {
let (readers, filters) = split_readers(client, &request).await?;
let (batches, mut queued) =
tokio::sync::mpsc::channel::<Result<RecordBatch, PostgresError>>(COPY_QUEUE_BATCHES);
let reads = start_reads(&readers, filters, &request, &batches).await?;
drop(batches);
let consumed = async {
while let Some(batch) = queued.recv().await {
on_batch(batch?).await.map_err(PostgresError::Consumer)?;
}
Ok(0)
}
.await;
drop(queued);
let outcome = join_reads(consumed, reads).await;
match (readers.len(), outcome) {
(2.., Ok(total)) => end_snapshot(&readers).await.map(|()| total),
(_, outcome) => outcome,
}
}