use std::fs::File;
use arrow::array::{Array, Int64Array, RecordBatch};
use arrow::ipc::reader::StreamReader;
use serde_json::{json, Value as J};
#[derive(Clone)]
pub struct ArrowCfg {
pub dataset: String,
pub path: String,
pub partitions: usize,
}
pub fn parse(dataset: &str, ds: &J) -> Option<Result<ArrowCfg, String>> {
if ds["connector"].as_str() != Some("arrow") {
return None;
}
let path = match ds["path"].as_str() {
Some(p) => p.to_string(),
None => {
return Some(Err(format!(
"pipeline.toml: [datasets.{dataset}] connector = \"arrow\" needs `path = \"…\"`"
)))
}
};
Some(Ok(ArrowCfg {
dataset: dataset.to_string(),
path,
partitions: ds["partitions"].as_i64().unwrap_or(4).max(1) as usize,
}))
}
pub struct ArrowInline {
pub cfg: ArrowCfg,
}
impl fv_streams_engine::InlineSource for ArrowInline {
fn name(&self) -> String {
format!("arrow:{}", self.cfg.dataset)
}
fn tasks(&self) -> usize {
self.cfg.partitions.max(1)
}
fn open(
&self,
task: usize,
tasks: usize,
start: Option<i64>,
watermark_column: Option<&str>,
) -> Box<dyn fv_streams_engine::dataflow::Source + Send> {
let file = File::open(&self.cfg.path)
.unwrap_or_else(|e| panic!("arrow source open {}: {e}", self.cfg.path));
let reader = StreamReader::try_new(file, None)
.unwrap_or_else(|e| panic!("arrow source {}: {e}", self.cfg.path));
let batches: Vec<RecordBatch> = reader
.collect::<Result<_, _>>()
.unwrap_or_else(|e| panic!("arrow source {}: {e}", self.cfg.path));
let mut base = 0i64;
let offsets: Vec<i64> = batches
.iter()
.map(|b| {
let o = base;
base += b.num_rows() as i64;
o
})
.collect();
Box::new(ArrowFileSource {
batches,
offsets,
next: start.map(|s| s.max(0) as usize).unwrap_or(0),
task,
tasks,
watermark_column: watermark_column.map(String::from),
done: false,
})
}
}
struct ArrowFileSource {
batches: Vec<RecordBatch>,
offsets: Vec<i64>,
next: usize,
task: usize,
tasks: usize,
watermark_column: Option<String>,
done: bool,
}
impl fv_streams_engine::dataflow::Source for ArrowFileSource {
fn poll(&mut self, out: &mut fv_streams_engine::dataflow::Out) -> fv_streams_engine::dataflow::Poll {
use fv_streams_engine::dataflow::Poll;
if self.done {
return Poll::Done;
}
while self.next < self.batches.len() {
let i = self.next;
self.next += 1;
if i % self.tasks != self.task {
continue;
}
let batch = &self.batches[i];
if batch.num_rows() == 0 {
continue;
}
let base = self.offsets[i];
let keys: Vec<Option<String>> = (0..batch.num_rows()).map(|r| Some((base + r as i64).to_string())).collect();
let stamped = fv_streams_engine::decode::with_partition(
&fv_streams_engine::decode::with_keys(batch, &keys),
self.task as i32,
);
if let Some(wc) = &self.watermark_column {
if let Some(max) = max_i64(batch, wc) {
out.watermark = Some(Some(max));
}
}
out.push(stamped);
return Poll::Again;
}
self.done = true;
Poll::Done
}
fn on_barrier(&mut self, _epoch: u64) -> Vec<u8> {
serde_json::to_vec(&json!({ "in": { self.task.to_string(): self.next } })).unwrap_or_default()
}
}
fn max_i64(batch: &RecordBatch, column: &str) -> Option<i64> {
let col = batch.column_by_name(column)?;
let a = col.as_any().downcast_ref::<Int64Array>()?;
(0..a.len()).filter(|&i| a.is_valid(i)).map(|i| a.value(i)).max()
}