use std::sync::Arc;
use std::time::{Duration, Instant};
use arrow_array::RecordBatch;
use arrow_schema::SchemaRef;
use tokio::sync::mpsc::Sender;
use crate::python_sources::models::{TransformCall, TransformRan};
use crate::python_sources::types::{PythonHost, Streamed};
pub(in crate::python_sources) async fn run_transform<E>(
host: Arc<dyn PythonHost>,
call: TransformCall,
(rows, schema): (Vec<RecordBatch>, SchemaRef),
mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), E>,
) -> Result<Result<TransformRan, String>, E> {
let rows_in = rows.iter().map(RecordBatch::num_rows).sum::<usize>() as u64;
let (sender, mut receiver) = tokio::sync::mpsc::channel::<RecordBatch>(1);
let crossing =
tokio::task::spawn_blocking(move || cross(host.as_ref(), &call, (rows, schema), &sender));
while let Some(batch) = receiver.recv().await {
if let Err(error) = on_batch(batch).await {
drop(receiver);
let _ = crossing.await;
return Err(error);
}
}
let crossed = match crossing.await {
Ok(crossed) => crossed,
Err(error) => Err(format!("the transform's thread failed: {error}")),
};
Ok(crossed.map(|(dicts, spent)| TransformRan {
dicts,
rows_in,
seconds: spent.as_secs_f64(),
}))
}
fn cross(
host: &dyn PythonHost,
call: &TransformCall,
(rows, schema): (Vec<RecordBatch>, SchemaRef),
sender: &Sender<RecordBatch>,
) -> Result<(bool, Duration), String> {
let started = Instant::now();
let mut waited = Duration::ZERO;
let mut stream = host.transform(call, rows, schema)?;
while let Some(next) = stream.next_batch() {
if let Streamed::Batch(batch) = next.map_err(|error| error.message)? {
let waiting = Instant::now();
if sender.blocking_send(batch).is_err() {
break;
}
waited += waiting.elapsed();
}
}
Ok((
stream.crossed_dicts(),
started.elapsed().saturating_sub(waited),
))
}