use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use tokio::io::{AsyncWriteExt, BufReader, BufWriter, ReadHalf, WriteHalf};
use tokio::sync::{mpsc, oneshot};
use tokio::task::JoinHandle;
use tokio::time::Sleep;
use crate::error::{Error, Result};
use crate::tcp::client_info::ClientInfo;
use crate::tcp::connect;
use crate::tcp::protocol::ServerHello;
use crate::tcp::reader::{self, ServerPacket};
use crate::tcp::transport::MaybeTlsStream;
use crate::tcp::writer::{self, CLIENT_NAME, CLIENT_VERSION_MAJOR_STR, CLIENT_VERSION_MINOR_STR};
use crate::worker::{self, CommandWorker, WorkerControl, WorkerHandle};
const PACKET_CHANNEL_CAPACITY: usize = 4;
const IDLE_TIMER_PARK: Duration = Duration::from_secs(3600);
const DEFAULT_CMD_CHANNEL: usize = 16;
const STREAM_CHANNEL_CAPACITY: usize = 4;
const DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
const MAX_INSERT_BLOCK_BYTES: usize = 512 * 1024 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ActorState {
Idle,
InsertActive,
}
pub(crate) enum ConnectionCmd {
Ping { reply: oneshot::Sender<Result<()>> },
ExecuteQuery {
query_id: String,
query: String,
extra_settings: Vec<(String, String)>,
params: Vec<(String, String)>,
reply: oneshot::Sender<Result<()>>,
},
BeginInsert {
query_id: String,
query: String,
extra_settings: Vec<(String, String)>,
params: Vec<(String, String)>,
reply: oneshot::Sender<Result<Vec<(String, String)>>>,
},
SendInsertBlock {
column_bytes: Vec<u8>,
num_columns: u64,
num_rows: u64,
reply: oneshot::Sender<Result<()>>,
},
FinishInsert { reply: oneshot::Sender<Result<()>> },
ExecuteStream {
query_id: String,
query: String,
extra_settings: Vec<(String, String)>,
params: Vec<(String, String)>,
results: mpsc::Sender<Result<ServerPacket>>,
},
}
#[derive(Clone)]
#[non_exhaustive]
pub struct ConnectionHandle {
inner: WorkerHandle<ConnectionCmd>,
server_hello: Arc<ServerHello>,
poisoned: Arc<AtomicBool>,
control: Arc<tokio::sync::Mutex<Option<WorkerControl<ConnectionCmd>>>>,
}
impl ConnectionHandle {
pub async fn close(self) -> Result<()> {
let control = self.control.lock().await.take();
match control {
Some(control) => control.shutdown().await.map_err(|e| {
Error::Custom(format!(
"tcp: connection actor task did not exit cleanly: {e}"
))
}),
None => Ok(()),
}
}
#[must_use]
pub fn is_alive(&self) -> bool {
!self.poisoned.load(Ordering::Acquire)
}
pub(crate) fn poison(&self) {
self.poisoned.store(true, Ordering::Release);
}
pub(crate) fn server_hello(&self) -> &ServerHello {
&self.server_hello
}
pub async fn ping(&self) -> Result<()> {
let (reply, rx) = oneshot::channel();
self.inner
.send(ConnectionCmd::Ping { reply })
.await
.map_err(|_| Error::Custom("tcp: connection actor closed".into()))?;
rx.await
.map_err(|_| Error::Custom("tcp: connection actor dropped ping reply".into()))?
}
pub async fn execute_query(
&self,
query_id: String,
query: String,
extra_settings: Vec<(String, String)>,
params: Vec<(String, String)>,
) -> Result<()> {
let (reply, rx) = oneshot::channel();
self.inner
.send(ConnectionCmd::ExecuteQuery {
query_id,
query,
extra_settings,
params,
reply,
})
.await
.map_err(|_| Error::Custom("tcp: connection actor closed".into()))?;
rx.await.map_err(|_| {
Error::Custom("tcp: connection actor dropped execute_query reply".into())
})?
}
pub async fn begin_insert(
&self,
query_id: String,
query: String,
extra_settings: Vec<(String, String)>,
params: Vec<(String, String)>,
) -> Result<Vec<(String, String)>> {
let (reply, rx) = oneshot::channel();
self.inner
.send(ConnectionCmd::BeginInsert {
query_id,
query,
extra_settings,
params,
reply,
})
.await
.map_err(|_| Error::Custom("tcp: connection actor closed".into()))?;
rx.await
.map_err(|_| Error::Custom("tcp: connection actor dropped begin_insert reply".into()))?
}
pub async fn send_insert_block(
&self,
column_bytes: Vec<u8>,
num_columns: u64,
num_rows: u64,
) -> Result<()> {
let (reply, rx) = oneshot::channel();
self.inner
.send(ConnectionCmd::SendInsertBlock {
column_bytes,
num_columns,
num_rows,
reply,
})
.await
.map_err(|_| Error::Custom("tcp: connection actor closed".into()))?;
rx.await.map_err(|_| {
Error::Custom("tcp: connection actor dropped send_insert_block reply".into())
})?
}
pub(crate) async fn execute_stream(
&self,
query_id: String,
query: String,
extra_settings: Vec<(String, String)>,
params: Vec<(String, String)>,
results: mpsc::Sender<Result<ServerPacket>>,
) -> Result<()> {
self.inner
.send(ConnectionCmd::ExecuteStream {
query_id,
query,
extra_settings,
params,
results,
})
.await
.map_err(|_| Error::Custom("tcp: connection actor closed".into()))
}
pub async fn execute_stream_cursor(
&self,
query_id: String,
query: String,
extra_settings: Vec<(String, String)>,
params: Vec<(String, String)>,
) -> Result<crate::tcp::cursor::TcpRawCursor> {
let (tx, rx) = mpsc::channel(STREAM_CHANNEL_CAPACITY);
self.execute_stream(query_id, query, extra_settings, params, tx)
.await?;
Ok(crate::tcp::cursor::TcpRawCursor::from_receiver(rx))
}
pub async fn finish_insert(&self) -> Result<()> {
let (reply, rx) = oneshot::channel();
self.inner
.send(ConnectionCmd::FinishInsert { reply })
.await
.map_err(|_| Error::Custom("tcp: connection actor closed".into()))?;
rx.await.map_err(|_| {
Error::Custom("tcp: connection actor dropped finish_insert reply".into())
})?
}
}
enum ReaderMessage {
Packet(ServerPacket),
Error(Error),
}
#[derive(Clone, Debug, Default)]
#[non_exhaustive]
pub struct ActorConfig {
pub read_timeout: Option<Duration>,
pub quota_key: String,
}
pub struct ConnectionActor {
writer: BufWriter<WriteHalf<MaybeTlsStream>>,
pkt_rx: mpsc::Receiver<ReaderMessage>,
reader_task: JoinHandle<()>,
server_hello: Arc<ServerHello>,
poisoned: Arc<AtomicBool>,
state: ActorState,
read_timeout: Option<Duration>,
quota_key: String,
idle_sleep: Pin<Box<Sleep>>,
}
enum PacketWait {
Idle,
Message(Option<ReaderMessage>),
}
enum QueryEvent {
CallerGone,
Idle,
Message(Option<ReaderMessage>),
}
impl ConnectionActor {
#[must_use]
pub fn spawn(stream: MaybeTlsStream, server_hello: ServerHello) -> ConnectionHandle {
Self::spawn_with_config(stream, server_hello, ActorConfig::default())
}
#[must_use]
pub fn spawn_with_config(
stream: MaybeTlsStream,
server_hello: ServerHello,
config: ActorConfig,
) -> ConnectionHandle {
let (reader_half, writer_half) = connect::split_buffered(stream);
let server_hello = Arc::new(server_hello);
let poisoned = Arc::new(AtomicBool::new(false));
let revision = server_hello.revision;
let (pkt_tx, pkt_rx) = mpsc::channel(PACKET_CHANNEL_CAPACITY);
let reader_task = tokio::spawn(reader_loop(reader_half, pkt_tx, revision));
let actor = ConnectionActor {
writer: writer_half,
pkt_rx,
reader_task,
server_hello: Arc::clone(&server_hello),
poisoned: Arc::clone(&poisoned),
state: ActorState::Idle,
read_timeout: config.read_timeout,
quota_key: config.quota_key,
idle_sleep: Box::pin(tokio::time::sleep(IDLE_TIMER_PARK)),
};
let control = worker::spawn(actor, DEFAULT_CMD_CHANNEL);
let inner = control.handle();
ConnectionHandle {
inner,
server_hello,
poisoned,
control: Arc::new(tokio::sync::Mutex::new(Some(control))),
}
}
fn arm_idle(&mut self) {
let gap = self.read_timeout.unwrap_or(IDLE_TIMER_PARK);
self.idle_sleep
.as_mut()
.reset(tokio::time::Instant::now() + gap);
}
async fn next_packet(&mut self) -> PacketWait {
self.arm_idle();
let Self {
idle_sleep, pkt_rx, ..
} = self;
tokio::select! {
biased;
() = idle_sleep.as_mut() => PacketWait::Idle,
msg = pkt_rx.recv() => PacketWait::Message(msg),
}
}
fn idle_timed_out(&self) -> Option<Error> {
self.read_timeout?;
self.poisoned.store(true, Ordering::Release);
Some(Error::TimedOut)
}
fn client_info_for(&self, query_id: &str) -> ClientInfo {
let mut info = ClientInfo::for_initial_query(
CLIENT_NAME,
CLIENT_VERSION_MAJOR_STR.parse().unwrap_or(0),
CLIENT_VERSION_MINOR_STR.parse().unwrap_or(0),
self.server_hello.revision,
&self.quota_key,
);
info.initial_query_id.push_str(query_id);
info
}
async fn issue_query(
&mut self,
query_id: &str,
query: &str,
extra_settings: &[(String, String)],
params: &[(String, String)],
) -> Result<()> {
let client_info = self.client_info_for(query_id);
let revision = self.server_hello.revision;
writer::send_query(
&mut self.writer,
revision,
query_id,
query,
extra_settings,
params,
&client_info,
)
.await?;
writer::send_empty_block(&mut self.writer, revision).await
}
async fn do_ping(&mut self) -> Result<()> {
writer::send_ping(&mut self.writer).await.inspect_err(|_| {
self.poisoned.store(true, Ordering::Release);
})?;
loop {
let msg = match self.next_packet().await {
PacketWait::Idle => match self.idle_timed_out() {
Some(e) => return Err(e),
None => continue,
},
PacketWait::Message(msg) => msg,
};
match msg {
Some(ReaderMessage::Packet(ServerPacket::Pong)) => return Ok(()),
Some(ReaderMessage::Packet(ServerPacket::Exception(exc))) => {
return Err(exc.into_error());
}
Some(ReaderMessage::Packet(other)) => {
tracing::trace!(
target: "clickhouse::tcp",
?other,
"ignoring interleaved packet during ping"
);
}
Some(ReaderMessage::Error(e)) => {
self.poisoned.store(true, Ordering::Release);
return Err(e);
}
None => {
self.poisoned.store(true, Ordering::Release);
return Err(Error::Custom("tcp: reader sub-task exited mid-ping".into()));
}
}
}
}
async fn do_execute_query(
&mut self,
query_id: String,
query: String,
extra_settings: Vec<(String, String)>,
params: Vec<(String, String)>,
mut reply: oneshot::Sender<Result<()>>,
) {
if let Err(e) = self
.issue_query(&query_id, &query, &extra_settings, ¶ms)
.await
{
self.poisoned.store(true, Ordering::Release);
let _ = reply.send(Err(e));
return;
}
loop {
self.arm_idle();
let event = {
let Self {
idle_sleep, pkt_rx, ..
} = &mut *self;
tokio::select! {
biased;
() = reply.closed() => QueryEvent::CallerGone,
() = idle_sleep.as_mut() => QueryEvent::Idle,
msg = pkt_rx.recv() => QueryEvent::Message(msg),
}
};
match event {
QueryEvent::CallerGone => {
self.cancel_and_drain("caller dropped the reply").await;
return;
}
QueryEvent::Idle => {
if let Some(e) = self.idle_timed_out() {
let _ = reply.send(Err(e));
return;
}
}
QueryEvent::Message(Some(ReaderMessage::Packet(ServerPacket::EndOfStream))) => {
let _ = reply.send(Ok(()));
return;
}
QueryEvent::Message(Some(ReaderMessage::Packet(ServerPacket::Exception(exc)))) => {
let _ = reply.send(Err(exc.into_error()));
return;
}
QueryEvent::Message(Some(ReaderMessage::Packet(_))) => {}
QueryEvent::Message(Some(ReaderMessage::Error(e))) => {
self.poisoned.store(true, Ordering::Release);
let _ = reply.send(Err(e));
return;
}
QueryEvent::Message(None) => {
self.poisoned.store(true, Ordering::Release);
let _ = reply.send(Err(Error::Custom(
"tcp: reader sub-task exited mid-query".into(),
)));
return;
}
}
}
}
async fn cancel_and_drain(&mut self, reason: &str) {
if let Err(e) = writer::send_cancel(&mut self.writer).await {
self.poisoned.store(true, Ordering::Release);
tracing::warn!(
target: "clickhouse::tcp",
error = %e,
reason,
"failed to send Cancel"
);
return;
}
if let Err(e) = self.drain_to_end_of_stream().await {
tracing::warn!(
target: "clickhouse::tcp",
error = %e,
reason,
"drain after Cancel did not reach EndOfStream"
);
}
}
async fn do_begin_insert(
&mut self,
query_id: String,
query: String,
extra_settings: Vec<(String, String)>,
params: Vec<(String, String)>,
) -> Result<Vec<(String, String)>> {
if let Err(e) = self
.issue_query(&query_id, &query, &extra_settings, ¶ms)
.await
{
self.poisoned.store(true, Ordering::Release);
return Err(e);
}
loop {
let msg = match self.next_packet().await {
PacketWait::Idle => match self.idle_timed_out() {
Some(e) => return Err(e),
None => continue,
},
PacketWait::Message(msg) => msg,
};
match msg {
Some(ReaderMessage::Packet(ServerPacket::Data { columns, .. })) => {
return Ok(columns);
}
Some(ReaderMessage::Packet(ServerPacket::DataBlock(block))) => {
self.poisoned.store(true, Ordering::Release);
return Err(Error::BadResponse(format!(
"tcp: server sent payload Data block (num_rows={}) \
before INSERT schema block",
block.num_rows
)));
}
Some(ReaderMessage::Packet(ServerPacket::Exception(exc))) => {
return Err(exc.into_error());
}
Some(ReaderMessage::Packet(ServerPacket::EndOfStream)) => {
return Err(Error::BadResponse(
"tcp: server returned EndOfStream before INSERT schema block \
(was this INSERT ... SELECT?)"
.into(),
));
}
Some(ReaderMessage::Packet(_)) => {}
Some(ReaderMessage::Error(e)) => {
self.poisoned.store(true, Ordering::Release);
return Err(e);
}
None => {
self.poisoned.store(true, Ordering::Release);
return Err(Error::Custom(
"tcp: reader sub-task exited mid begin_insert".into(),
));
}
}
}
}
async fn do_send_insert_block(
&mut self,
column_bytes: Vec<u8>,
num_columns: u64,
num_rows: u64,
reply: oneshot::Sender<Result<()>>,
) {
if column_bytes.len() > MAX_INSERT_BLOCK_BYTES {
let _ = reply.send(Err(Error::Custom(format!(
"tcp: INSERT block of {} bytes exceeds the {MAX_INSERT_BLOCK_BYTES} byte cap; \
split the batch into smaller blocks",
column_bytes.len()
))));
return;
}
loop {
match self.pkt_rx.try_recv() {
Ok(ReaderMessage::Packet(ServerPacket::Exception(exc))) => {
self.state = ActorState::Idle;
let _ = reply.send(Err(exc.into_error()));
return;
}
Ok(ReaderMessage::Packet(_)) => {}
Ok(ReaderMessage::Error(e)) => {
self.poisoned.store(true, Ordering::Release);
self.state = ActorState::Idle;
let _ = reply.send(Err(e));
return;
}
Err(mpsc::error::TryRecvError::Empty) => break,
Err(mpsc::error::TryRecvError::Disconnected) => {
self.poisoned.store(true, Ordering::Release);
self.state = ActorState::Idle;
let _ = reply.send(Err(Error::Custom(
"tcp: reader sub-task exited mid-INSERT".into(),
)));
return;
}
}
}
let result = writer::send_data_block(
&mut self.writer,
self.server_hello.revision,
"",
&column_bytes,
num_columns,
num_rows,
)
.await;
if result.is_err() {
self.poisoned.store(true, Ordering::Release);
self.state = ActorState::Idle;
}
let _ = reply.send(result);
}
async fn do_finish_insert(&mut self) -> Result<()> {
if let Err(e) = writer::send_empty_block(&mut self.writer, self.server_hello.revision).await
{
self.poisoned.store(true, Ordering::Release);
return Err(e);
}
self.drain_to_end_of_stream().await
}
async fn do_execute_stream(
&mut self,
query_id: String,
query: String,
extra_settings: Vec<(String, String)>,
params: Vec<(String, String)>,
results: mpsc::Sender<Result<ServerPacket>>,
) {
if let Err(e) = self
.issue_query(&query_id, &query, &extra_settings, ¶ms)
.await
{
self.poisoned.store(true, Ordering::Release);
let _ = results.send(Err(e)).await;
return;
}
loop {
self.arm_idle();
let event = {
let Self {
idle_sleep, pkt_rx, ..
} = &mut *self;
tokio::select! {
biased;
() = results.closed() => QueryEvent::CallerGone,
() = idle_sleep.as_mut() => QueryEvent::Idle,
msg = pkt_rx.recv() => QueryEvent::Message(msg),
}
};
match event {
QueryEvent::CallerGone => {
self.cancel_and_drain("stream receiver dropped").await;
return;
}
QueryEvent::Idle => {
if let Some(e) = self.idle_timed_out() {
let _ = results.send(Err(e)).await;
return;
}
}
QueryEvent::Message(Some(ReaderMessage::Packet(ServerPacket::EndOfStream))) => {
let _ = results.send(Ok(ServerPacket::EndOfStream)).await;
return;
}
QueryEvent::Message(Some(ReaderMessage::Packet(ServerPacket::Exception(exc)))) => {
let _ = results.send(Err(exc.into_error())).await;
return;
}
QueryEvent::Message(Some(ReaderMessage::Packet(
ServerPacket::Pong | ServerPacket::Log | ServerPacket::ProfileEvents,
))) => {}
QueryEvent::Message(Some(ReaderMessage::Packet(pkt))) => {
if results.send(Ok(pkt)).await.is_err() {
self.cancel_and_drain("stream receiver dropped mid-send")
.await;
return;
}
}
QueryEvent::Message(Some(ReaderMessage::Error(e))) => {
self.poisoned.store(true, Ordering::Release);
let _ = results.send(Err(e)).await;
return;
}
QueryEvent::Message(None) => {
self.poisoned.store(true, Ordering::Release);
let _ = results
.send(Err(Error::Custom(
"tcp: reader sub-task exited mid-stream".into(),
)))
.await;
return;
}
}
}
}
async fn drain_to_end_of_stream(&mut self) -> Result<()> {
let drain = async {
loop {
match self.pkt_rx.recv().await {
Some(ReaderMessage::Packet(ServerPacket::EndOfStream)) => return Ok(()),
Some(ReaderMessage::Packet(ServerPacket::Exception(exc))) => {
return Err(exc.into_error());
}
Some(ReaderMessage::Packet(_)) => {}
Some(ReaderMessage::Error(e)) => {
self.poisoned.store(true, Ordering::Release);
return Err(e);
}
None => {
self.poisoned.store(true, Ordering::Release);
return Err(Error::Custom(
"tcp: reader sub-task exited mid-drain".into(),
));
}
}
}
};
let Ok(drained) = tokio::time::timeout(DRAIN_TIMEOUT, drain).await else {
self.poisoned.store(true, Ordering::Release);
return Err(Error::Custom(
"tcp: drain to EndOfStream exceeded 30s timeout".into(),
));
};
drained
}
}
impl CommandWorker for ConnectionActor {
type Command = ConnectionCmd;
fn name() -> &'static str {
"clickhouse.tcp-connection"
}
async fn handle(&mut self, cmd: Self::Command) {
match (self.state, cmd) {
(
ActorState::InsertActive,
ConnectionCmd::Ping { reply } | ConnectionCmd::ExecuteQuery { reply, .. },
) => {
let _ = reply.send(Err(Error::Custom("tcp: actor busy in INSERT".into())));
}
(ActorState::InsertActive, ConnectionCmd::BeginInsert { reply, .. }) => {
let _ = reply.send(Err(Error::Custom("tcp: actor already in INSERT".into())));
}
(ActorState::InsertActive, ConnectionCmd::ExecuteStream { results, .. }) => {
let _ = results
.send(Err(Error::Custom("tcp: actor busy in INSERT".into())))
.await;
}
(
ActorState::Idle,
ConnectionCmd::SendInsertBlock { reply, .. }
| ConnectionCmd::FinishInsert { reply },
) => {
let _ = reply.send(Err(Error::Custom("tcp: no INSERT session active".into())));
}
(ActorState::Idle, ConnectionCmd::Ping { reply }) => {
let result = self.do_ping().await;
if reply.send(result).is_err() {
tracing::warn!(
target: "clickhouse::tcp",
"ping reply dropped by caller before send"
);
}
}
(
ActorState::Idle,
ConnectionCmd::ExecuteQuery {
query_id,
query,
extra_settings,
params,
reply,
},
) => {
self.do_execute_query(query_id, query, extra_settings, params, reply)
.await;
}
(
ActorState::Idle,
ConnectionCmd::BeginInsert {
query_id,
query,
extra_settings,
params,
reply,
},
) => {
let result = self
.do_begin_insert(query_id, query, extra_settings, params)
.await;
if result.is_ok() {
self.state = ActorState::InsertActive;
}
if reply.send(result).is_err() {
tracing::warn!(
target: "clickhouse::tcp",
"begin_insert reply dropped by caller before send"
);
}
}
(
ActorState::InsertActive,
ConnectionCmd::SendInsertBlock {
column_bytes,
num_columns,
num_rows,
reply,
},
) => {
self.do_send_insert_block(column_bytes, num_columns, num_rows, reply)
.await;
}
(ActorState::InsertActive, ConnectionCmd::FinishInsert { reply }) => {
let result = self.do_finish_insert().await;
self.state = ActorState::Idle;
if reply.send(result).is_err() {
tracing::warn!(
target: "clickhouse::tcp",
"finish_insert reply dropped by caller before send"
);
}
}
(
ActorState::Idle,
ConnectionCmd::ExecuteStream {
query_id,
query,
extra_settings,
params,
results,
},
) => {
self.do_execute_stream(query_id, query, extra_settings, params, results)
.await;
}
}
}
async fn on_shutdown(&mut self) {
if let Err(e) = self.writer.shutdown().await {
tracing::debug!(
target: "clickhouse::tcp",
error = %e,
"closing the writer half on shutdown failed"
);
}
}
}
impl Drop for ConnectionActor {
fn drop(&mut self) {
self.reader_task.abort();
if self.poisoned.load(Ordering::Acquire) {
tracing::warn!(
target: "clickhouse::tcp",
"connection actor dropped while poisoned"
);
} else {
tracing::debug!(
target: "clickhouse::tcp",
"connection actor dropped cleanly"
);
}
}
}
async fn reader_loop(
mut r: BufReader<ReadHalf<MaybeTlsStream>>,
tx: mpsc::Sender<ReaderMessage>,
server_revision: u64,
) {
loop {
match reader::read_packet(&mut r, server_revision).await {
Ok(pkt) => {
if tx.send(ReaderMessage::Packet(pkt)).await.is_err() {
return;
}
}
Err(e) => {
let _ = tx.send(ReaderMessage::Error(e)).await;
return;
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::native::io::ClickHouseWrite;
use crate::tcp::mock::{write_schema_block, write_uint64_payload_block};
use crate::tcp::protocol::{ClientPacketId, DBMS_TCP_PROTOCOL_VERSION, ServerPacketId};
use tokio::io::AsyncWriteExt;
use tokio::net::{TcpListener, TcpStream};
static_assertions::assert_impl_all!(ConnectionHandle: Send, Sync);
async fn paired() -> (ConnectionHandle, TcpStream) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let connect_fut = TcpStream::connect(addr);
let accept_fut = async { listener.accept().await.unwrap().0 };
let (client, server) = tokio::join!(connect_fut, accept_fut);
let client = client.unwrap();
let _ = client.set_nodelay(true);
let _ = server.set_nodelay(true);
let stream = MaybeTlsStream::Plain(client);
let hello = ServerHello {
server_name: "mock".to_string(),
version: (1, 0, 0),
revision: DBMS_TCP_PROTOCOL_VERSION,
timezone: None,
display_name: None,
};
let handle = ConnectionActor::spawn(stream, hello);
(handle, server)
}
async fn paired_with_config(config: ActorConfig) -> (ConnectionHandle, TcpStream) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let connect_fut = TcpStream::connect(addr);
let accept_fut = async { listener.accept().await.unwrap().0 };
let (client, server) = tokio::join!(connect_fut, accept_fut);
let client = client.unwrap();
let _ = client.set_nodelay(true);
let _ = server.set_nodelay(true);
let stream = MaybeTlsStream::Plain(client);
let hello = ServerHello {
server_name: "mock".to_string(),
version: (1, 0, 0),
revision: DBMS_TCP_PROTOCOL_VERSION,
timezone: None,
display_name: None,
};
let handle = ConnectionActor::spawn_with_config(stream, hello, config);
(handle, server)
}
async fn read_byte(server: &mut TcpStream) -> u8 {
let mut buf = [0u8; 1];
tokio::io::AsyncReadExt::read_exact(server, &mut buf)
.await
.unwrap();
buf[0]
}
#[tokio::test]
async fn ping_happy_path() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let id = read_byte(&mut server).await;
assert_eq!(u64::from(id), ClientPacketId::Ping as u64);
server
.write_var_uint(ServerPacketId::Pong as u64)
.await
.unwrap();
server.flush().await.unwrap();
server
});
handle.ping().await.expect("ping should succeed");
assert!(handle.is_alive());
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn ping_during_reader_disconnect() {
let (handle, server) = paired().await;
drop(server);
let result = handle.ping().await;
assert!(result.is_err(), "ping should fail after server hangup");
assert!(
!handle.is_alive(),
"handle should be poisoned after reader-task exit"
);
}
#[tokio::test]
async fn dropping_the_last_handle_closes_the_socket_and_exits_the_reader() {
use tokio::io::AsyncReadExt;
let (handle, mut server) = paired().await;
drop(handle);
let mut buf = [0u8; 64];
let n = tokio::time::timeout(Duration::from_secs(2), server.read(&mut buf))
.await
.expect("the socket must close promptly, not wait on a server timeout")
.expect("read after client close");
assert_eq!(n, 0, "expected EOF on the server side, got {n} bytes");
}
#[tokio::test]
async fn close_waits_for_the_actor_and_is_idempotent() {
use tokio::io::AsyncReadExt;
let (handle, mut server) = paired().await;
let second = handle.clone();
handle.close().await.expect("close should succeed");
let mut buf = [0u8; 64];
let n = tokio::time::timeout(Duration::from_secs(2), server.read(&mut buf))
.await
.expect("close must have shut the socket down")
.expect("read after close");
assert_eq!(n, 0);
second.close().await.expect("second close should be Ok");
}
#[tokio::test(start_paused = true)]
async fn drain_timeout_poisons_when_cancel_is_ignored() {
use tokio::io::AsyncReadExt;
let (handle, mut server) = paired().await;
let cancel_handle = handle.clone();
let exec = tokio::spawn(async move {
cancel_handle
.execute_query(
"q_drain".into(),
"SELECT sleep(9)".into(),
Vec::new(),
Vec::new(),
)
.await
});
let mut buf = [0u8; 4096];
let n = server.read(&mut buf).await.expect("the Query packet");
assert!(n > 0, "the actor must have issued the query");
exec.abort();
for _ in 0..10 {
if !handle.is_alive() {
break;
}
tokio::time::advance(DRAIN_TIMEOUT).await;
tokio::task::yield_now().await;
}
assert!(
!handle.is_alive(),
"a drain that never reaches EndOfStream must poison the connection"
);
drop(server);
}
#[tokio::test]
async fn dropping_the_cursor_cancels_and_leaves_the_connection_reusable() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
use tokio::io::AsyncReadExt;
let _drained = drain_client_bytes(&mut server).await;
write_schema_block(&mut server, &[("n", "UInt64")]).await;
write_uint64_payload_block(&mut server, &[1, 2, 3]).await;
let mut chunk = [0u8; 4096];
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
let mut saw_cancel = false;
while tokio::time::Instant::now() < deadline {
let left = deadline.saturating_duration_since(tokio::time::Instant::now());
match tokio::time::timeout(left, server.read(&mut chunk)).await {
Ok(Ok(0)) => break,
Ok(Ok(n)) => {
if chunk[..n].contains(&(ClientPacketId::Cancel as u8)) {
saw_cancel = true;
break;
}
}
_ => break,
}
}
assert!(saw_cancel, "dropping the cursor must send Cancel");
server
.write_var_uint(ServerPacketId::EndOfStream as u64)
.await
.unwrap();
server.flush().await.unwrap();
let id = read_byte(&mut server).await;
assert_eq!(u64::from(id), ClientPacketId::Ping as u64);
server
.write_var_uint(ServerPacketId::Pong as u64)
.await
.unwrap();
server.flush().await.unwrap();
server
});
let mut cursor = handle
.execute_stream_cursor(
"rs_drop".into(),
"SELECT number AS n FROM numbers(3)".into(),
Vec::new(),
Vec::new(),
)
.await
.expect("execute_stream_cursor should succeed");
let schema = cursor.next_block().await.unwrap().expect("schema block");
assert_eq!(schema.num_rows, 0);
drop(cursor);
handle
.ping()
.await
.expect("the connection must be reusable after a cursor drop");
assert!(handle.is_alive());
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn send_insert_block_rejects_oversized_block_and_stays_in_insert() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
write_schema_block(&mut server, &[("n", "UInt64")]).await;
tokio::time::sleep(Duration::from_millis(400)).await;
server
});
handle
.begin_insert(
"q_big".into(),
"INSERT INTO t FORMAT Native".into(),
Vec::new(),
Vec::new(),
)
.await
.expect("begin_insert should succeed");
let err = handle
.send_insert_block(vec![0u8; MAX_INSERT_BLOCK_BYTES + 1], 1, 1)
.await
.expect_err("an oversized block must be refused");
match err {
Error::Custom(msg) => assert!(msg.contains("exceeds the"), "got {msg}"),
other => panic!("expected Custom, got {other:?}"),
}
handle
.send_insert_block(Vec::new(), 1, 0)
.await
.expect("the session must survive a refused block");
assert!(handle.is_alive());
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn execute_stream_during_insert_yields_one_err_frame() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
write_schema_block(&mut server, &[("n", "UInt64")]).await;
tokio::time::sleep(Duration::from_millis(400)).await;
server
});
handle
.begin_insert(
"q_busy_stream".into(),
"INSERT INTO t FORMAT Native".into(),
Vec::new(),
Vec::new(),
)
.await
.expect("begin_insert should succeed");
let mut cursor = handle
.execute_stream_cursor("rs_busy".into(), "SELECT 1".into(), Vec::new(), Vec::new())
.await
.expect("dispatch succeeds; the misuse surfaces on the cursor");
let err = cursor
.next_block()
.await
.expect_err("a SELECT during an INSERT must surface an error");
match err {
Error::Custom(msg) => assert!(msg.contains("busy in INSERT"), "got {msg}"),
other => panic!("expected Custom, got {other:?}"),
}
assert!(handle.is_alive(), "misuse must not poison the connection");
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn begin_insert_exception_leaves_actor_idle_and_alive() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
write_server_exception(&mut server, 60, "no such table").await;
let id = read_byte(&mut server).await;
assert_eq!(u64::from(id), ClientPacketId::Ping as u64);
server
.write_var_uint(ServerPacketId::Pong as u64)
.await
.unwrap();
server.flush().await.unwrap();
server
});
let err = handle
.begin_insert(
"q_reject".into(),
"INSERT INTO nope FORMAT Native".into(),
Vec::new(),
Vec::new(),
)
.await
.expect_err("the server rejected the INSERT");
match err {
Error::ServerException { code, .. } => assert_eq!(code, 60),
other => panic!("expected ServerException, got {other:?}"),
}
assert!(handle.is_alive(), "a rejection must not poison");
handle
.ping()
.await
.expect("the actor must be back in Idle after a rejected INSERT");
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn double_drop_safety() {
let (handle, mut server) = paired().await;
let clone = handle.clone();
let ping_fut = tokio::spawn(async move { clone.ping().await });
let server_task = tokio::spawn(async move {
let _ = read_byte(&mut server).await;
tokio::time::sleep(Duration::from_millis(20)).await;
server
.write_var_uint(ServerPacketId::Pong as u64)
.await
.unwrap();
server.flush().await.unwrap();
server
});
drop(handle);
let ping_result = ping_fut.await.unwrap();
assert!(
ping_result.is_ok(),
"ping should still succeed: {ping_result:?}"
);
let _server = server_task.await.unwrap();
}
async fn drain_client_bytes(server: &mut TcpStream) -> Vec<u8> {
use tokio::io::AsyncReadExt;
let mut buf = Vec::new();
let mut chunk = [0u8; 4096];
let deadline = tokio::time::Instant::now() + Duration::from_millis(200);
loop {
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
if remaining.is_zero() {
break;
}
match tokio::time::timeout(remaining, server.read(&mut chunk)).await {
Ok(Ok(n)) if n > 0 => buf.extend_from_slice(&chunk[..n]),
_ => break,
}
}
buf
}
#[tokio::test]
async fn execute_query_happy_path() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
server
.write_var_uint(ServerPacketId::EndOfStream as u64)
.await
.unwrap();
server.flush().await.unwrap();
server
});
let result = handle
.execute_query("q1".into(), "SELECT 1".into(), Vec::new(), Vec::new())
.await;
assert!(result.is_ok(), "execute_query should succeed: {result:?}");
assert!(handle.is_alive());
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn execute_query_surfaces_server_exception() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
server
.write_var_uint(ServerPacketId::Exception as u64)
.await
.unwrap();
server.write_i32_le(60i32).await.unwrap();
server
.write_string("DB::Exception".as_bytes())
.await
.unwrap();
server
.write_string("table not found".as_bytes())
.await
.unwrap();
server.write_string("".as_bytes()).await.unwrap();
tokio::io::AsyncWriteExt::write_u8(&mut server, 0u8)
.await
.unwrap(); server.flush().await.unwrap();
server
});
let err = handle
.execute_query(
"q2".into(),
"SELECT * FROM nope".into(),
Vec::new(),
Vec::new(),
)
.await
.expect_err("expected server Exception to surface");
match err {
Error::ServerException { code, .. } => assert_eq!(code, 60),
other => panic!("expected ServerException, got {other:?}"),
}
assert!(
handle.is_alive(),
"connection should remain alive after a server Exception"
);
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn execute_query_cancel_on_reply_drop() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
use tokio::io::AsyncReadExt;
let mut initial = Vec::new();
let mut chunk = [0u8; 4096];
loop {
match tokio::time::timeout(Duration::from_millis(50), server.read(&mut chunk)).await
{
Ok(Ok(0)) => break,
Ok(Ok(n)) => initial.extend_from_slice(&chunk[..n]),
_ => break,
}
}
assert!(
!initial.is_empty(),
"server should have seen at least the Query packet"
);
let mut saw_cancel = false;
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
while tokio::time::Instant::now() < deadline {
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
match tokio::time::timeout(remaining, server.read(&mut chunk)).await {
Ok(Ok(0)) => break,
Ok(Ok(n)) => {
if chunk[..n].contains(&(ClientPacketId::Cancel as u8)) {
saw_cancel = true;
break;
}
}
_ => break,
}
}
assert!(saw_cancel, "server should have observed Cancel byte");
server
.write_var_uint(ServerPacketId::EndOfStream as u64)
.await
.unwrap();
server.flush().await.unwrap();
server
});
let cancel_handle = handle.clone();
let exec_fut = cancel_handle.execute_query(
"qcancel".into(),
"SELECT sleep(10)".into(),
Vec::new(),
Vec::new(),
);
let timeout_result = tokio::time::timeout(Duration::from_millis(150), exec_fut).await;
assert!(
timeout_result.is_err(),
"execute_query should not have completed inside the timeout"
);
let mut server = server_task.await.unwrap();
assert!(
handle.is_alive(),
"handle should remain alive after a successful cancel-and-drain"
);
let ping_server_task = tokio::spawn(async move {
let id = read_byte(&mut server).await;
assert_eq!(u64::from(id), ClientPacketId::Ping as u64);
server
.write_var_uint(ServerPacketId::Pong as u64)
.await
.unwrap();
server.flush().await.unwrap();
server
});
handle
.ping()
.await
.expect("connection should be reusable after cancel");
let _server = ping_server_task.await.unwrap();
}
async fn write_server_exception(server: &mut TcpStream, code: i32, message: &str) {
server
.write_var_uint(ServerPacketId::Exception as u64)
.await
.unwrap();
server.write_i32_le(code).await.unwrap();
server.write_string(b"DB::Exception").await.unwrap();
server.write_string(message.as_bytes()).await.unwrap();
server.write_string(b"").await.unwrap();
AsyncWriteExt::write_u8(server, 0).await.unwrap(); server.flush().await.unwrap();
}
#[tokio::test]
async fn insert_state_machine_rejects_ping_when_busy() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
write_schema_block(&mut server, &[("n", "UInt64")]).await;
tokio::time::sleep(Duration::from_millis(200)).await;
server
});
let headers = handle
.begin_insert(
"q_busy".into(),
"INSERT INTO t FORMAT Native".into(),
Vec::new(),
Vec::new(),
)
.await
.expect("begin_insert should succeed against scripted server");
assert_eq!(headers, vec![("n".to_string(), "UInt64".to_string())]);
let err = handle
.ping()
.await
.expect_err("ping during InsertActive must error");
match err {
Error::Custom(msg) => assert!(
msg.contains("busy"),
"expected 'busy' in ping error, got {msg}"
),
other => panic!("expected Custom busy error, got {other:?}"),
}
assert!(handle.is_alive());
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn send_insert_block_without_begin_errs() {
let (handle, _server) = paired().await;
let err = handle
.send_insert_block(Vec::new(), 0, 0)
.await
.expect_err("send_insert_block in Idle must error");
match err {
Error::Custom(msg) => assert!(
msg.contains("no INSERT session"),
"expected 'no INSERT session' in error, got {msg}"
),
other => panic!("expected Custom error, got {other:?}"),
}
}
#[tokio::test]
async fn finish_insert_returns_to_idle() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
write_schema_block(&mut server, &[("n", "UInt64")]).await;
let _block_bytes = drain_client_bytes(&mut server).await;
server
.write_var_uint(ServerPacketId::EndOfStream as u64)
.await
.unwrap();
server.flush().await.unwrap();
let id = read_byte(&mut server).await;
assert_eq!(u64::from(id), ClientPacketId::Ping as u64);
server
.write_var_uint(ServerPacketId::Pong as u64)
.await
.unwrap();
server.flush().await.unwrap();
server
});
let headers = handle
.begin_insert(
"q_finish".into(),
"INSERT INTO t FORMAT Native".into(),
Vec::new(),
Vec::new(),
)
.await
.expect("begin_insert should succeed");
assert_eq!(headers.len(), 1);
handle
.send_insert_block(Vec::new(), 1, 0)
.await
.expect("send_insert_block should succeed");
handle
.finish_insert()
.await
.expect("finish_insert should return Ok on EndOfStream");
handle
.ping()
.await
.expect("connection should be reusable after finish_insert");
assert!(handle.is_alive());
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn full_duplex_exception_aborts_send() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
write_schema_block(&mut server, &[("n", "UInt64")]).await;
tokio::time::sleep(Duration::from_millis(20)).await;
write_server_exception(&mut server, 241, "MEMORY_LIMIT_EXCEEDED").await;
server
});
handle
.begin_insert(
"q_fd".into(),
"INSERT INTO t FORMAT Native".into(),
Vec::new(),
Vec::new(),
)
.await
.expect("begin_insert should succeed");
tokio::time::sleep(Duration::from_millis(100)).await;
let err = handle
.send_insert_block(Vec::new(), 1, 0)
.await
.expect_err("send_insert_block must surface the queued Exception");
match err {
Error::ServerException { code, .. } => assert_eq!(code, 241),
other => panic!("expected ServerException(241), got {other:?}"),
}
assert!(handle.is_alive());
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn execute_stream_yields_schema_then_payload_then_eos() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
write_schema_block(&mut server, &[("n", "UInt64")]).await;
write_uint64_payload_block(&mut server, &[10, 20, 30]).await;
server
.write_var_uint(ServerPacketId::EndOfStream as u64)
.await
.unwrap();
server.flush().await.unwrap();
server
});
let mut cursor = handle
.execute_stream_cursor(
"rs_stream_unit".into(),
"SELECT number AS n FROM numbers(3)".into(),
Vec::new(),
Vec::new(),
)
.await
.expect("execute_stream_cursor should succeed");
let schema_block = cursor
.next_block()
.await
.expect("schema block decode")
.expect("schema block");
assert_eq!(schema_block.num_rows, 0);
assert_eq!(
schema_block.schema,
vec![("n".to_string(), "UInt64".to_string())]
);
let payload = cursor
.next_block()
.await
.expect("payload decode")
.expect("payload block");
assert_eq!(payload.num_rows, 3);
match &payload.columns[0] {
crate::native::DecodedColumn::UInt64(values) => {
assert_eq!(values, &vec![10u64, 20, 30]);
}
other => panic!("expected UInt64, got {other:?}"),
}
assert!(cursor.next_block().await.unwrap().is_none());
assert!(cursor.next_block().await.unwrap().is_none());
assert!(handle.is_alive());
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn execute_stream_surfaces_server_exception() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
write_server_exception(&mut server, 60, "table not found").await;
server
});
let mut cursor = handle
.execute_stream_cursor(
"rs_stream_err".into(),
"SELECT * FROM doesnt_exist".into(),
Vec::new(),
Vec::new(),
)
.await
.expect("execute_stream_cursor should succeed");
let err = cursor
.next_block()
.await
.expect_err("server Exception must surface");
match err {
Error::ServerException { code, .. } => assert_eq!(code, 60),
other => panic!("expected ServerException, got {other:?}"),
}
assert!(cursor.next_block().await.unwrap().is_none());
assert!(handle.is_alive());
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn execute_query_read_timeout_poisons() {
let (handle, mut server) = paired_with_config(ActorConfig {
read_timeout: Some(Duration::from_millis(80)),
..ActorConfig::default()
})
.await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
tokio::time::sleep(Duration::from_millis(500)).await;
server
});
let err = handle
.execute_query(
"rq_timeout".into(),
"SELECT sleep(9)".into(),
Vec::new(),
Vec::new(),
)
.await
.expect_err("a silent server must surface TimedOut, not hang");
assert!(matches!(err, Error::TimedOut), "got {err:?}");
assert!(err.is_retriable(), "TimedOut must be retriable");
assert!(!handle.is_alive(), "timed-out connection must be poisoned");
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn execute_stream_read_timeout_poisons() {
let (handle, mut server) = paired_with_config(ActorConfig {
read_timeout: Some(Duration::from_millis(80)),
..ActorConfig::default()
})
.await;
let server_task = tokio::spawn(async move {
write_schema_block(&mut server, &[("n", "UInt64")]).await;
tokio::time::sleep(Duration::from_millis(500)).await;
server
});
let mut cursor = handle
.execute_stream_cursor(
"rs_timeout".into(),
"SELECT number AS n FROM numbers(9)".into(),
Vec::new(),
Vec::new(),
)
.await
.expect("execute_stream_cursor should succeed");
let schema = cursor.next_block().await.unwrap().expect("schema block");
assert_eq!(schema.num_rows, 0);
let err = cursor
.next_block()
.await
.expect_err("silent server mid-stream must surface TimedOut");
assert!(matches!(err, Error::TimedOut), "got {err:?}");
assert!(!handle.is_alive(), "timed-out connection must be poisoned");
let _server = server_task.await.unwrap();
}
#[tokio::test]
async fn execute_stream_aligns_rows_across_multiple_blocks() {
let (handle, mut server) = paired().await;
let server_task = tokio::spawn(async move {
let _drained = drain_client_bytes(&mut server).await;
write_schema_block(&mut server, &[("n", "UInt64")]).await;
write_uint64_payload_block(&mut server, &[10, 20, 30]).await;
write_uint64_payload_block(&mut server, &[40, 50]).await;
server
.write_var_uint(ServerPacketId::EndOfStream as u64)
.await
.unwrap();
server.flush().await.unwrap();
server
});
let mut cursor = handle
.execute_stream_cursor(
"rs_multiblock".into(),
"SELECT number AS n FROM numbers(5)".into(),
Vec::new(),
Vec::new(),
)
.await
.expect("execute_stream_cursor should succeed");
let schema = cursor.next_block().await.unwrap().expect("schema block");
assert_eq!(schema.num_rows, 0);
let b1 = cursor.next_block().await.unwrap().expect("first payload");
assert_eq!(b1.num_rows, 3);
match &b1.columns[0] {
crate::native::DecodedColumn::UInt64(v) => assert_eq!(v, &vec![10u64, 20, 30]),
other => panic!("expected UInt64, got {other:?}"),
}
let b2 = cursor.next_block().await.unwrap().expect("second payload");
assert_eq!(b2.num_rows, 2);
match &b2.columns[0] {
crate::native::DecodedColumn::UInt64(v) => assert_eq!(v, &vec![40u64, 50]),
other => panic!("expected UInt64, got {other:?}"),
}
assert!(cursor.next_block().await.unwrap().is_none());
assert!(handle.is_alive());
let _server = server_task.await.unwrap();
}
}