use std::fs::File;
use arrow::array::{Array, Int64Array, RecordBatch};
use arrow::ipc::reader::StreamReader;
use serde_json::Value as J;
#[derive(Clone, Debug)]
pub struct ArrowCfg {
pub dataset: String,
pub path: String,
pub partitions: usize,
}
pub fn read_output(path: &str) -> Result<Vec<RecordBatch>, String> {
let p = std::path::Path::new(path);
let files: Vec<std::path::PathBuf> = if p.is_dir() {
fv_streams_engine::file_sink::committed_parts(p).map_err(|e| e.to_string())?
} else {
vec![p.to_path_buf()]
};
let mut out = Vec::new();
for f in files {
let file = File::open(&f).map_err(|e| format!("cannot open {}: {e}", f.display()))?;
let reader = StreamReader::try_new(file, None).map_err(|e| format!("{}: {e}", f.display()))?;
for b in reader {
out.push(b.map_err(|e| format!("{}: {e}", f.display()))?);
}
}
Ok(out)
}
pub fn parse(dataset: &str, ds: &J) -> Option<Result<ArrowCfg, String>> {
if ds["connector"].as_str() != Some("arrow") {
return None;
}
let path = match ds["path"].as_str() {
Some(p) => p.to_string(),
None => {
return Some(Err(format!(
"pipeline.toml: [datasets.{dataset}] connector = \"arrow\" needs `path = \"…\"`"
)))
}
};
let p = std::path::Path::new(&path);
let readable = if p.is_dir() {
fv_streams_engine::file_sink::read_manifest(p)
.map(|_| ())
.map_err(|e| e.to_string())
} else {
File::open(p)
.map_err(|e| e.to_string())
.and_then(|f| StreamReader::try_new(f, None).map(|_| ()).map_err(|e| e.to_string()))
};
if let Err(e) = readable {
return Some(Err(format!(
"pipeline.toml: [datasets.{dataset}] connector = \"arrow\": cannot read {path}: {e}"
)));
}
Some(Ok(ArrowCfg {
dataset: dataset.to_string(),
path,
partitions: ds["partitions"].as_i64().unwrap_or(4).max(1) as usize,
}))
}
pub struct ArrowInline {
pub cfg: ArrowCfg,
}
impl fv_streams_engine::InlineSource for ArrowInline {
fn name(&self) -> String {
format!("arrow:{}", self.cfg.dataset)
}
fn tasks(&self) -> usize {
self.cfg.partitions.max(1)
}
fn open(
&self,
task: usize,
tasks: usize,
start: Option<i64>,
watermark_column: Option<&str>,
) -> Result<Box<dyn fv_streams_engine::dataflow::Source + Send>, String> {
let path = &self.cfg.path;
let batches = read_output(path).map_err(|e| format!("arrow source {path}: {e}"))?;
let mut base = 0i64;
let offsets: Vec<i64> = batches
.iter()
.map(|b| {
let o = base;
base += b.num_rows() as i64;
o
})
.collect();
Ok(Box::new(ArrowFileSource {
name: self.name(),
batches,
offsets,
next: start.map(|s| s.max(0) as usize).unwrap_or(0),
task,
tasks,
watermark_column: watermark_column.map(String::from),
done: false,
}))
}
}
struct ArrowFileSource {
name: String,
batches: Vec<RecordBatch>,
offsets: Vec<i64>,
next: usize,
task: usize,
tasks: usize,
watermark_column: Option<String>,
done: bool,
}
impl fv_streams_engine::dataflow::Source for ArrowFileSource {
fn poll(&mut self, out: &mut fv_streams_engine::dataflow::Out) -> fv_streams_engine::dataflow::Poll {
use fv_streams_engine::dataflow::Poll;
if self.done {
return Poll::Done;
}
while self.next < self.batches.len() {
let i = self.next;
self.next += 1;
if i % self.tasks != self.task {
continue;
}
let batch = &self.batches[i];
if batch.num_rows() == 0 {
continue;
}
let base = self.offsets[i];
let keys: Vec<Option<String>> = (0..batch.num_rows())
.map(|r| Some((base + r as i64).to_string()))
.collect();
let stamped = fv_streams_engine::decode::with_partition(
&fv_streams_engine::decode::with_keys(batch, &keys),
self.task as i32,
);
if let Some(wc) = &self.watermark_column {
if let Some(max) = max_i64(batch, wc) {
out.watermark = Some(Some(max));
}
}
out.push(stamped);
return Poll::Again;
}
self.done = true;
Poll::Done
}
fn on_barrier(&mut self, _epoch: u64) -> Vec<u8> {
fv_streams_engine::dataflow::position_record(&self.name, self.task as i32, self.next as i64)
}
fn position(&mut self) -> Vec<u8> {
fv_streams_engine::dataflow::position_record(&self.name, self.task as i32, self.next as i64)
}
}
fn max_i64(batch: &RecordBatch, column: &str) -> Option<i64> {
let col = batch.column_by_name(column)?;
let a = col.as_any().downcast_ref::<Int64Array>()?;
(0..a.len()).filter(|&i| a.is_valid(i)).map(|i| a.value(i)).max()
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn the_position_record_names_the_source_and_a_restore_there_reads_nothing_twice() {
use arrow::datatypes::{DataType, Field, Schema};
use fv_streams_engine::dataflow::{Out, Poll};
use fv_streams_engine::InlineSource;
use std::sync::Arc;
let p = std::env::temp_dir().join(format!("fv-arrow-source-pos-{}.arrow", std::process::id()));
let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, false)]));
let mut w = arrow::ipc::writer::StreamWriter::try_new(File::create(&p).unwrap(), &schema).unwrap();
for i in 0..3 {
w.write(&RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![i]))]).unwrap())
.unwrap();
}
w.finish().unwrap();
let src = ArrowInline {
cfg: ArrowCfg {
dataset: "events".into(),
path: p.to_string_lossy().to_string(),
partitions: 1,
},
};
let drain = |start: Option<i64>| -> (usize, Vec<u8>) {
let mut s = src.open(0, 1, start, None).unwrap();
let mut rows = 0;
loop {
let mut out = Out::default();
match s.poll(&mut out) {
Poll::Again | Poll::Idle => rows += out.batches.iter().map(|b| b.num_rows()).sum::<usize>(),
Poll::Done => break (rows, s.position()),
Poll::Failed(e) => panic!("{e}"),
}
}
};
let (rows, pos) = drain(None);
assert_eq!(rows, 3);
let v: serde_json::Value = serde_json::from_slice(&pos).unwrap();
assert_eq!(v["topic"], "arrow:events");
assert_eq!(v["in"]["0"], 3);
assert_eq!(drain(Some(3)).0, 0, "restored at the end: nothing again");
assert_eq!(drain(Some(1)).0, 2);
std::fs::remove_file(&p).ok();
}
#[test]
fn parse_validates_the_file_up_front_so_a_bad_source_fails_the_build_not_a_task() {
let ds = json!({ "connector": "arrow", "path": "/nonexistent/events.arrow" });
let err = parse("events", &ds)
.expect("recognised as an arrow source")
.unwrap_err();
assert!(
err.contains("[datasets.events]") && err.contains("cannot read /nonexistent/events.arrow"),
"{err}"
);
let p = std::env::temp_dir().join(format!("fv-arrow-source-{}.bin", std::process::id()));
std::fs::write(&p, b"not an arrow ipc stream").unwrap();
let ds = json!({ "connector": "arrow", "path": p.to_string_lossy() });
let err = parse("events", &ds).expect("recognised").unwrap_err();
std::fs::remove_file(&p).ok();
assert!(err.contains("cannot read"), "{err}");
assert!(parse("k", &json!({ "connector": "kafka" })).is_none());
}
}