use crate::clock;
use crate::error::{Error, TypedError};
use crate::memory_limiter::MemoryPressureChanged;
use crate::message::Sender;
use crate::node::{NodeId, NodeType};
use crate::shared::message::{SharedReceiver, SharedSender};
use bytemuck::Pod;
use otel_arrow_dfe_channel::error::SendError;
use otel_arrow_dfe_telemetry::reporter::MetricsReporter;
use smallvec::SmallVec;
use std::collections::HashMap;
use std::marker::PhantomData;
use std::time::{Duration, Instant};
#[must_use]
pub fn nanos_since_birth() -> u64 {
clock::nanos_since_birth()
}
#[repr(transparent)]
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Context8u8([u8; 8]);
impl<T: Pod> From<T> for Context8u8 {
fn from(v: T) -> Self {
const {
assert!(size_of::<T>() == 8);
}
Self(bytemuck::cast(v))
}
}
impl TryFrom<Context8u8> for usize {
type Error = Error;
fn try_from(v: Context8u8) -> Result<Self, Self::Error> {
bytemuck::try_cast(v.0).map_err(|_| Error::InternalError {
message: "bytecast error".into(),
})
}
}
impl From<Context8u8> for u64 {
fn from(v: Context8u8) -> u64 {
bytemuck::cast(v.0)
}
}
impl From<Context8u8> for f64 {
fn from(v: Context8u8) -> f64 {
bytemuck::cast(v.0)
}
}
pub type CallData = SmallVec<[Context8u8; 3]>;
#[repr(transparent)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct WakeupSlot(pub u128);
pub type WakeupRevision = u64;
#[derive(Clone, Debug, Default, PartialEq)]
pub struct RouteData {
pub calldata: CallData,
pub entry_time_ns: u64,
pub output_port_index: u16,
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct UnwindData {
pub route: RouteData,
pub return_time_ns: u64,
}
impl UnwindData {
#[must_use]
pub fn new(route: RouteData, return_time_ns: u64) -> Self {
Self {
route,
return_time_ns,
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct Frame {
pub interests: crate::Interests,
pub route: RouteData,
pub node_id: usize,
pub output_items: u32,
pub input_items: u32,
pub output_size: u64,
pub input_size: u64,
}
#[derive(Debug, Clone)]
pub struct AckMsg<PData> {
pub accepted: Box<PData>,
pub unwind: UnwindData,
}
impl<PData> AckMsg<PData> {
pub fn new(accepted: PData) -> Self {
Self {
accepted: Box::new(accepted),
unwind: UnwindData::default(),
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum NackCause {
#[default]
Unspecified,
RouteFull,
RouteClosed,
NodeShutdown,
Refused,
}
#[derive(Debug, Clone)]
pub struct NackMsg<PData> {
pub reason: String,
pub refused: Box<PData>,
pub unwind: UnwindData,
pub permanent: bool,
pub cause: NackCause,
}
impl<PData> NackMsg<PData> {
pub fn new<T: Into<String>>(reason: T, refused: PData) -> Self {
Self::new_internal(reason, refused, false, NackCause::Unspecified)
}
pub fn new_with_cause<T: Into<String>>(reason: T, refused: PData, cause: NackCause) -> Self {
Self::new_internal(reason, refused, false, cause)
}
pub fn new_permanent<T: Into<String>>(reason: T, refused: PData) -> Self {
Self::new_internal(reason, refused, true, NackCause::Unspecified)
}
pub fn new_permanent_with_cause<T: Into<String>>(
reason: T,
refused: PData,
cause: NackCause,
) -> Self {
Self::new_internal(reason, refused, true, cause)
}
fn new_internal<T: Into<String>>(
reason: T,
refused: PData,
permanent: bool,
cause: NackCause,
) -> Self {
Self {
reason: reason.into(),
refused: Box::new(refused),
unwind: UnwindData::default(),
permanent,
cause,
}
}
}
#[derive(Debug, Clone)]
pub enum NodeControlMsg<PData> {
Ack(AckMsg<PData>),
Nack(NackMsg<PData>),
Config {
config: serde_json::Value,
},
TimerTick {
},
CollectTelemetry {
metrics_reporter: MetricsReporter,
},
Wakeup {
slot: WakeupSlot,
when: Instant,
revision: WakeupRevision,
},
ResumeData {
when: Instant,
data: Box<PData>,
},
MemoryPressureChanged {
update: MemoryPressureChanged,
},
DrainIngress {
deadline: Instant,
reason: String,
},
Shutdown {
deadline: Instant,
reason: String,
},
}
#[derive(Debug, Clone)]
pub enum RuntimeControlMsg<PData> {
StartTimer {
node_id: usize,
duration: Duration,
},
CancelTimer {
node_id: usize,
},
StartTelemetryTimer {
node_id: usize,
duration: Duration,
},
CancelTelemetryTimer {
node_id: usize,
_temp: PhantomData<PData>,
},
ReceiverDrained {
node_id: usize,
},
Shutdown {
deadline: Instant,
reason: String,
},
}
#[derive(Debug, Clone)]
pub enum PipelineCompletionMsg<PData> {
DeliverAck {
ack: AckMsg<PData>,
},
DeliverNack {
nack: NackMsg<PData>,
},
}
#[async_trait::async_trait(?Send)]
pub trait Controllable<PData> {
fn control_sender(&self) -> Sender<NodeControlMsg<PData>>;
}
impl<PData> NodeControlMsg<PData> {
#[must_use]
pub const fn is_shutdown(&self) -> bool {
matches!(self, NodeControlMsg::Shutdown { .. })
}
#[must_use]
pub const fn is_drain_ingress(&self) -> bool {
matches!(self, NodeControlMsg::DrainIngress { .. })
}
}
pub type RuntimeCtrlMsgSender<PData> = SharedSender<RuntimeControlMsg<PData>>;
pub type RuntimeCtrlMsgReceiver<PData> = SharedReceiver<RuntimeControlMsg<PData>>;
pub type PipelineCompletionMsgSender<PData> = SharedSender<PipelineCompletionMsg<PData>>;
pub type PipelineCompletionMsgReceiver<PData> = SharedReceiver<PipelineCompletionMsg<PData>>;
pub trait PipelineAdminSender: Send + Sync {
fn try_send_shutdown(&self, deadline: Instant, reason: String) -> Result<(), Error>;
}
pub fn runtime_ctrl_msg_channel<PData>(
capacity: usize,
) -> (RuntimeCtrlMsgSender<PData>, RuntimeCtrlMsgReceiver<PData>) {
let (tx, rx) = tokio::sync::mpsc::channel(capacity);
(SharedSender::mpsc(tx), SharedReceiver::mpsc(rx))
}
pub fn pipeline_completion_msg_channel<PData>(
capacity: usize,
) -> (
PipelineCompletionMsgSender<PData>,
PipelineCompletionMsgReceiver<PData>,
) {
let (tx, rx) = tokio::sync::mpsc::channel(capacity);
(SharedSender::mpsc(tx), SharedReceiver::mpsc(rx))
}
#[derive(Clone)]
pub struct TypedControlSender<PData> {
pub node_id: NodeId,
pub node_type: NodeType,
pub sender: Sender<NodeControlMsg<PData>>,
}
#[derive(Clone)]
pub struct ControlSenders<PData> {
senders: HashMap<usize, TypedControlSender<PData>>,
}
impl<PData> TypedControlSender<PData> {
#[inline]
pub async fn send(
&self,
msg: NodeControlMsg<PData>,
) -> Result<(), SendError<NodeControlMsg<PData>>> {
self.sender.send(msg).await
}
#[inline]
pub fn try_send(
&self,
msg: NodeControlMsg<PData>,
) -> Result<(), SendError<NodeControlMsg<PData>>> {
self.sender.try_send(msg)
}
}
impl<PData> Default for ControlSenders<PData> {
fn default() -> Self {
Self::new()
}
}
impl<PData> ControlSenders<PData> {
#[must_use]
pub fn new() -> Self {
Self {
senders: HashMap::new(),
}
}
#[must_use]
pub fn get(&self, node_id: usize) -> Option<&TypedControlSender<PData>> {
self.senders.get(&node_id)
}
pub fn register(
&mut self,
node_id: NodeId,
node_type: NodeType,
sender: Sender<NodeControlMsg<PData>>,
) {
_ = self.senders.insert(
node_id.index,
TypedControlSender {
node_id,
node_type,
sender,
},
);
}
#[must_use]
pub fn len(&self) -> usize {
self.senders.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.senders.is_empty()
}
#[must_use]
pub fn receiver_count(&self) -> usize {
self.senders
.values()
.filter(|sender| sender.node_type == NodeType::Receiver)
.count()
}
#[must_use]
pub fn receiver_ids(&self) -> Vec<usize> {
self.senders
.values()
.filter(|sender| sender.node_type == NodeType::Receiver)
.map(|sender| sender.node_id.index)
.collect()
}
#[must_use]
pub fn node_ids(&self) -> Vec<usize> {
self.senders.keys().copied().collect()
}
#[must_use]
pub fn non_receiver_ids(&self) -> Vec<usize> {
self.senders
.values()
.filter(|sender| sender.node_type != NodeType::Receiver)
.map(|sender| sender.node_id.index)
.collect()
}
pub async fn drain_receivers(
&self,
deadline: Instant,
reason: String,
) -> Result<(), Vec<TypedError<NodeControlMsg<PData>>>> {
let mut errors: Vec<TypedError<NodeControlMsg<PData>>> = Vec::new();
for typed_sender in self.senders.values() {
if typed_sender.node_type != NodeType::Receiver {
continue;
}
if let Err(error) = typed_sender
.sender
.send(NodeControlMsg::DrainIngress {
deadline,
reason: reason.clone(),
})
.await
{
errors.push(TypedError::NodeControlMsgSendError {
node_id: typed_sender.node_id.index,
error,
});
}
}
if errors.is_empty() {
Ok(())
} else {
Err(errors)
}
}
pub async fn shutdown_receivers(
&self,
deadline: Instant,
reason: String,
) -> Result<(), Vec<TypedError<NodeControlMsg<PData>>>> {
self.shutdown_nodes(Some(NodeType::Receiver), deadline, reason)
.await
}
pub async fn shutdown_all(
&self,
deadline: Instant,
reason: String,
) -> Result<(), Vec<TypedError<NodeControlMsg<PData>>>> {
self.shutdown_nodes(None, deadline, reason).await
}
async fn shutdown_nodes(
&self,
node_type_filter: Option<NodeType>,
deadline: Instant,
reason: String,
) -> Result<(), Vec<TypedError<NodeControlMsg<PData>>>> {
let mut errors: Vec<TypedError<NodeControlMsg<PData>>> = Vec::new();
for typed_sender in self.senders.values() {
if let Some(filter_type) = node_type_filter
&& typed_sender.node_type != filter_type
{
continue;
}
let shutdown_msg = NodeControlMsg::Shutdown {
deadline,
reason: reason.clone(),
};
if let Err(error) = typed_sender.sender.send(shutdown_msg).await {
errors.push(TypedError::NodeControlMsgSendError {
node_id: typed_sender.node_id.index,
error,
});
}
}
if errors.is_empty() {
Ok(())
} else {
Err(errors)
}
}
}
impl<PData> PipelineAdminSender for SharedSender<RuntimeControlMsg<PData>>
where
PData: Send + Sync + 'static,
{
fn try_send_shutdown(&self, deadline: Instant, reason: String) -> Result<(), Error> {
let shutdown_msg = RuntimeControlMsg::Shutdown { deadline, reason };
self.try_send(shutdown_msg)
.map_err(|e| Error::RuntimeMsgError {
error: format!("Failed to send shutdown message: {}", e),
})
}
}
#[derive(Debug, Clone)]
pub enum ExtensionControlMsg {
Config {
config: serde_json::Value,
},
CollectTelemetry {
metrics_reporter: MetricsReporter,
},
Shutdown {
deadline: Instant,
reason: String,
},
}
impl ExtensionControlMsg {
#[must_use]
pub const fn is_shutdown(&self) -> bool {
matches!(self, ExtensionControlMsg::Shutdown { .. })
}
}
#[derive(Clone)]
pub struct ExtensionControlSender {
pub(crate) sender: Sender<ExtensionControlMsg>,
}
impl ExtensionControlSender {
pub async fn send(
&self,
msg: ExtensionControlMsg,
) -> Result<(), SendError<ExtensionControlMsg>> {
self.sender.send(msg).await
}
}
#[derive(Debug, Clone)]
pub struct ShutdownPayload {
pub deadline: Instant,
pub reason: String,
}
pub(crate) struct ExtensionShutdownChannel {
pub(crate) name: otel_arrow_dfe_config::ExtensionId,
pub(crate) sender: tokio::sync::oneshot::Sender<ShutdownPayload>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_permanent_status() {
assert!(!NackMsg::new("just bad news", ()).permanent);
assert!(NackMsg::new_permanent("very bad news", ()).permanent);
assert_eq!(
NackMsg::new("just bad news", ()).cause,
NackCause::Unspecified
);
assert_eq!(
NackMsg::new_permanent("very bad news", ()).cause,
NackCause::Unspecified
);
}
}