use std::collections::HashMap;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use arrow_flight::flight_service_server::{FlightService, FlightServiceServer};
use arrow_flight::{
Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo, HandshakeRequest, HandshakeResponse,
PollInfo, PutResult, SchemaResult, Ticket,
};
use futures::stream;
use serde_json::json;
use tonic::transport::Server;
use tonic::{Request, Response, Status, Streaming};
use crate::Pipeline;
type FlightStream<T> = Pin<Box<dyn futures::Stream<Item = Result<T, Status>> + Send + 'static>>;
type QueryRegistry = Arc<Mutex<HashMap<String, Arc<AtomicBool>>>>;
pub struct SqlService {
catalog: serde_json::Value,
host: String,
queries: QueryRegistry,
}
impl SqlService {
pub fn new(catalog: &Pipeline, host: &str) -> Result<Self, String> {
if catalog.def.get("sql").is_some() || catalog.def.get("transforms").is_some() {
return Err("a flight-sql catalog declares only [datasets.*] (+ [settings]); the SQL comes from each submitted statement".into());
}
Ok(SqlService {
catalog: catalog.def.clone(),
host: host.to_string(),
queries: Arc::new(Mutex::new(HashMap::new())),
})
}
async fn spawn_query(&self, sql: &str) -> Result<String, String> {
let stmt = sql.trim().trim_end_matches(';').trim();
let head = stmt.split_whitespace().next().unwrap_or("").to_ascii_uppercase();
if !matches!(head.as_str(), "SELECT" | "WITH") {
return Err("submit a `SELECT …` (optionally `WITH …`) — its result streams back to you".into());
}
let wrapped = format!("INSERT INTO __flight_result {stmt}");
fv_streams_sql::compile(&wrapped).map_err(|e| format!("SQL: {e}"))?;
let port = free_port()?;
let mut def = self.catalog.clone();
def["sql"] = json!(wrapped);
def["datasets"]["__flight_result"] = json!({ "sink": "flight", "host": self.host, "port": port });
let pipeline = Pipeline {
name: format!("flightsql-{port}"),
path: "<flight-sql>".to_string(),
def,
};
let endpoint = format!("grpc://{}:{port}", self.host);
let stop = Arc::new(AtomicBool::new(false));
self.queries
.lock()
.expect("queries")
.insert(endpoint.clone(), stop.clone());
let name = pipeline.name.clone();
let queries = Arc::clone(&self.queries);
let key = endpoint.clone();
std::thread::Builder::new()
.name(name.clone())
.spawn(move || {
let rt = match tokio::runtime::Builder::new_multi_thread().enable_all().build() {
Ok(rt) => rt,
Err(e) => {
eprintln!("flight-sql: query runtime for {name}: {e}");
queries.lock().expect("queries").remove(&key);
return;
}
};
if let Err(e) = rt.block_on(pipeline.run(stop)) {
eprintln!("flight-sql: query pipeline {name} ended: {e}");
}
queries.lock().expect("queries").remove(&key);
})
.map_err(|e| format!("flight-sql: spawn query thread: {e}"))?;
wait_listening(&self.host, port, Duration::from_secs(10)).await?;
Ok(endpoint)
}
}
#[tonic::async_trait]
impl FlightService for SqlService {
type HandshakeStream = FlightStream<HandshakeResponse>;
type ListFlightsStream = FlightStream<FlightInfo>;
type DoGetStream = FlightStream<FlightData>;
type DoPutStream = FlightStream<PutResult>;
type DoActionStream = FlightStream<arrow_flight::Result>;
type ListActionsStream = FlightStream<ActionType>;
type DoExchangeStream = FlightStream<FlightData>;
async fn do_action(&self, request: Request<Action>) -> Result<Response<Self::DoActionStream>, Status> {
let action = request.into_inner();
let body_str = || String::from_utf8(action.body.to_vec());
let reply = match action.r#type.as_str() {
"query" => {
let sql = body_str().map_err(|_| Status::invalid_argument("the query body must be UTF-8 SQL"))?;
self.spawn_query(&sql).await.map_err(Status::invalid_argument)?
}
"cancel" => {
let endpoint =
body_str().map_err(|_| Status::invalid_argument("the cancel body must be the endpoint string"))?;
match self.queries.lock().expect("queries").remove(endpoint.trim()) {
Some(flag) => {
flag.store(true, Ordering::Relaxed);
format!("cancelled {}", endpoint.trim())
}
None => return Err(Status::not_found(format!("no running query at {}", endpoint.trim()))),
}
}
other => {
return Err(Status::unimplemented(format!(
"unknown action '{other}' — use 'query' (body = SQL) or 'cancel' (body = the endpoint)"
)))
}
};
let out = stream::once(async move {
Ok(arrow_flight::Result {
body: reply.into_bytes().into(),
})
});
Ok(Response::new(Box::pin(out)))
}
async fn list_actions(&self, _: Request<Empty>) -> Result<Response<Self::ListActionsStream>, Status> {
let actions = vec![
ActionType {
r#type: "query".to_string(),
description:
"Compile and run a SQL statement; returns the grpc:// Flight endpoint of its Arrow result stream."
.to_string(),
},
ActionType {
r#type: "cancel".to_string(),
description: "Stop a running query; the body is the endpoint the 'query' action returned.".to_string(),
},
];
Ok(Response::new(Box::pin(stream::iter(actions.into_iter().map(Ok)))))
}
async fn handshake(
&self,
_: Request<Streaming<HandshakeRequest>>,
) -> Result<Response<Self::HandshakeStream>, Status> {
Err(Status::unimplemented(
"the flight-sql control endpoint takes a 'query' do_action",
))
}
async fn list_flights(&self, _: Request<Criteria>) -> Result<Response<Self::ListFlightsStream>, Status> {
Err(Status::unimplemented(
"the flight-sql control endpoint takes a 'query' do_action",
))
}
async fn get_flight_info(&self, _: Request<FlightDescriptor>) -> Result<Response<FlightInfo>, Status> {
Err(Status::unimplemented(
"the flight-sql control endpoint takes a 'query' do_action",
))
}
async fn poll_flight_info(&self, _: Request<FlightDescriptor>) -> Result<Response<PollInfo>, Status> {
Err(Status::unimplemented(
"the flight-sql control endpoint takes a 'query' do_action",
))
}
async fn get_schema(&self, _: Request<FlightDescriptor>) -> Result<Response<SchemaResult>, Status> {
Err(Status::unimplemented(
"the flight-sql control endpoint takes a 'query' do_action",
))
}
async fn do_get(&self, _: Request<Ticket>) -> Result<Response<Self::DoGetStream>, Status> {
Err(Status::unimplemented(
"read results from the endpoint the 'query' action returns",
))
}
async fn do_put(&self, _: Request<Streaming<FlightData>>) -> Result<Response<Self::DoPutStream>, Status> {
Err(Status::unimplemented(
"the flight-sql control endpoint takes a 'query' do_action",
))
}
async fn do_exchange(&self, _: Request<Streaming<FlightData>>) -> Result<Response<Self::DoExchangeStream>, Status> {
Err(Status::unimplemented(
"the flight-sql control endpoint takes a 'query' do_action",
))
}
}
pub async fn serve(catalog: Pipeline, host: &str, port: u16, shutdown: Arc<AtomicBool>) -> Result<(), String> {
let svc = SqlService::new(&catalog, host)?;
let queries = Arc::clone(&svc.queries); let addr = format!("{host}:{port}")
.parse()
.map_err(|e| format!("flight-sql: bad address {host}:{port}: {e}"))?;
println!("fv-streams: flight-sql control endpoint on grpc://{addr} — submit a 'query' action with your SQL");
let poll = async move {
while !shutdown.load(Ordering::Relaxed) {
tokio::time::sleep(Duration::from_millis(200)).await;
}
};
let result = Server::builder()
.add_service(FlightServiceServer::new(svc))
.serve_with_shutdown(addr, poll)
.await;
for (_, flag) in queries.lock().expect("queries").drain() {
flag.store(true, Ordering::Relaxed);
}
result.map_err(|e| format!("flight-sql server: {e}"))
}
fn free_port() -> Result<u16, String> {
let l = std::net::TcpListener::bind("127.0.0.1:0").map_err(|e| format!("flight-sql: no free port: {e}"))?;
l.local_addr()
.map(|a| a.port())
.map_err(|e| format!("flight-sql: local_addr: {e}"))
}
async fn wait_listening(host: &str, port: u16, within: Duration) -> Result<(), String> {
let deadline = tokio::time::Instant::now() + within;
loop {
if tokio::net::TcpStream::connect((host, port)).await.is_ok() {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!(
"the query's Flight output did not bind {host}:{port} within {within:?}"
));
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
#[cfg(test)]
mod tests {
use super::*;
use arrow::array::Int64Array;
use futures::StreamExt;
#[tokio::test]
async fn submit_a_sql_statement_and_read_its_arrow_result_over_flight() {
let catalog = Pipeline::parse(
r#"
name = "catalog"
[settings]
STREAM_STATE_STORE = "memory"
STREAM_CONSUMERS = "1"
STREAM_HEARTBEAT_SECONDS = "1"
[brokers]
default = "localhost:9092"
[datasets.events]
connector = "nexmark"
inline = true
events = 4000
rate = 1500 # pace the source so the live result stream stays up for the client to read
seed = 42
partitions = 1
"#,
"catalog.toml",
)
.unwrap();
let shutdown = Arc::new(AtomicBool::new(false));
let ctl_port = free_port().unwrap();
let server = {
let shutdown = shutdown.clone();
tokio::spawn(async move { serve(catalog, "127.0.0.1", ctl_port, shutdown).await })
};
wait_listening("127.0.0.1", ctl_port, Duration::from_secs(10))
.await
.unwrap();
let mut ctl = arrow_flight::FlightClient::new(
tonic::transport::Channel::from_shared(format!("http://127.0.0.1:{ctl_port}"))
.unwrap()
.connect()
.await
.unwrap(),
);
let mut results = ctl
.do_action(Action {
r#type: "query".to_string(),
body: "SELECT auction, price FROM events WHERE type = 'bid'".into(),
})
.await
.unwrap();
let endpoint = {
let body = results.next().await.unwrap().unwrap();
String::from_utf8(body.to_vec()).unwrap()
};
assert!(endpoint.starts_with("grpc://127.0.0.1:"), "got endpoint {endpoint}");
let data_addr = endpoint.strip_prefix("grpc://").unwrap();
let mut data = arrow_flight::FlightClient::new(
tonic::transport::Channel::from_shared(format!("http://{data_addr}"))
.unwrap()
.connect()
.await
.unwrap(),
);
let mut stream = data
.do_get(Ticket {
ticket: Default::default(),
})
.await
.unwrap();
let mut rows = 0usize;
let mut saw_auction = false;
while let Some(b) = stream.next().await {
let batch = b.unwrap();
rows += batch.num_rows();
if let Some(c) = batch.column_by_name("auction") {
saw_auction = c.as_any().downcast_ref::<Int64Array>().is_some();
}
}
assert!(rows > 0, "the submitted query streamed some bids back");
assert!(saw_auction, "the result carries the projected `auction` column");
shutdown.store(true, Ordering::Relaxed);
let _ = server.await;
}
#[tokio::test]
async fn a_running_query_can_be_cancelled() {
let catalog = Pipeline::parse(
r#"
name = "catalog"
[settings]
STREAM_STATE_STORE = "memory"
STREAM_CONSUMERS = "1"
STREAM_HEARTBEAT_SECONDS = "1"
STREAM_BATCH_MAX = "100" # small batches so the paced source returns from poll and sees a stop
[brokers]
default = "localhost:9092"
[datasets.events]
connector = "nexmark"
inline = true
events = 10000000
rate = 1000 # a long-running paced source (10000s at this rate), responsive to stop
seed = 42
partitions = 1
"#,
"catalog.toml",
)
.unwrap();
let shutdown = Arc::new(AtomicBool::new(false));
let ctl_port = free_port().unwrap();
let server = {
let shutdown = shutdown.clone();
tokio::spawn(async move { serve(catalog, "127.0.0.1", ctl_port, shutdown).await })
};
wait_listening("127.0.0.1", ctl_port, Duration::from_secs(10))
.await
.unwrap();
let mut ctl = arrow_flight::FlightClient::new(
tonic::transport::Channel::from_shared(format!("http://127.0.0.1:{ctl_port}"))
.unwrap()
.connect()
.await
.unwrap(),
);
let endpoint = {
let mut r = ctl
.do_action(Action {
r#type: "query".to_string(),
body: "SELECT * FROM events WHERE type = 'bid'".into(),
})
.await
.unwrap();
String::from_utf8(r.next().await.unwrap().unwrap().to_vec()).unwrap()
};
let addr = endpoint.strip_prefix("grpc://").unwrap().to_string();
assert!(
tokio::net::TcpStream::connect(&addr).await.is_ok(),
"the query is running"
);
let reply = {
let mut r = ctl
.do_action(Action {
r#type: "cancel".to_string(),
body: endpoint.clone().into(),
})
.await
.unwrap();
String::from_utf8(r.next().await.unwrap().unwrap().to_vec()).unwrap()
};
assert!(reply.starts_with("cancelled"), "got {reply}");
let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
loop {
if tokio::net::TcpStream::connect(&addr).await.is_err() {
break; }
assert!(
tokio::time::Instant::now() < deadline,
"the cancelled query kept serving {addr}"
);
tokio::time::sleep(Duration::from_millis(100)).await;
}
let err = ctl
.do_action(Action {
r#type: "cancel".to_string(),
body: endpoint.clone().into(),
})
.await
.err()
.expect("second cancel is not_found");
let not_found = matches!(&err, arrow_flight::error::FlightError::Tonic(s) if s.code() == tonic::Code::NotFound);
assert!(not_found, "expected not_found, got {err:?}");
shutdown.store(true, Ordering::Relaxed);
let _ = server.await;
}
}