use crate::capability::factory::{LocalInstanceFactory, SharedInstanceFactory};
use crate::channel_metrics::ChannelMetricsRegistry;
use crate::channel_mode::{LocalMode, SharedMode, wrap_extension_control_channel_metrics};
use crate::config::ExtensionConfig;
use crate::context::ExtensionContext;
use crate::control::{ExtensionControlMsg, ShutdownPayload};
use crate::entity_context::{EntityTelemetryGuard, EntityTelemetryHandle};
use crate::error::Error;
use crate::local::extension as local_ext;
use crate::local::message::LocalReceiver;
use crate::message::Sender;
use crate::shared::extension as shared_ext;
use crate::shared::message::SharedReceiver;
use crate::terminal_state::TerminalState;
use otel_arrow_dfe_channel::error::RecvError;
use otel_arrow_dfe_config::ExtensionId;
use otel_arrow_dfe_config::extension::ExtensionUserConfig;
use otel_arrow_dfe_telemetry::otel_debug;
use otel_arrow_dfe_telemetry::reporter::MetricsReporter;
use std::future::Future;
use std::sync::Arc;
#[doc(hidden)]
pub struct ControlChannel<R> {
control_rx: Option<R>,
shutdown_rx: Option<tokio::sync::oneshot::Receiver<ShutdownPayload>>,
}
#[doc(hidden)]
pub trait ControlReceiver {
fn recv(&mut self) -> impl Future<Output = Result<ExtensionControlMsg, RecvError>>;
}
impl ControlReceiver for LocalReceiver<ExtensionControlMsg> {
async fn recv(&mut self) -> Result<ExtensionControlMsg, RecvError> {
LocalReceiver::recv(self).await
}
}
impl ControlReceiver for SharedReceiver<ExtensionControlMsg> {
async fn recv(&mut self) -> Result<ExtensionControlMsg, RecvError> {
SharedReceiver::recv(self).await
}
}
impl<R: ControlReceiver + Unpin> ControlChannel<R> {
#[must_use]
pub fn new(
control_rx: R,
shutdown_rx: tokio::sync::oneshot::Receiver<ShutdownPayload>,
) -> Self {
ControlChannel {
control_rx: Some(control_rx),
shutdown_rx: Some(shutdown_rx),
}
}
pub async fn recv(&mut self) -> Result<ExtensionControlMsg, RecvError> {
loop {
let control_rx = self.control_rx.as_mut().ok_or(RecvError::Closed)?;
match self.shutdown_rx.as_mut() {
Some(shutdown_rx) => {
tokio::select! {
biased;
res = shutdown_rx => {
self.shutdown_rx = None;
match res {
Ok(payload) => {
let _ = self.control_rx.take();
return Ok(ExtensionControlMsg::Shutdown {
deadline: payload.deadline,
reason: payload.reason,
});
}
Err(_) => {
continue;
}
}
}
msg = control_rx.recv() => return msg,
}
}
None => return control_rx.recv().await,
}
}
}
}
#[derive(Clone)]
pub struct EffectHandler {
name: ExtensionId,
#[allow(dead_code)]
metrics_reporter: MetricsReporter,
readiness: Option<super::readiness::ReadinessSignaller>,
}
impl EffectHandler {
#[must_use]
pub(crate) fn new(
name: ExtensionId,
metrics_reporter: MetricsReporter,
readiness: Option<super::readiness::ReadinessSignaller>,
) -> Self {
EffectHandler {
name,
metrics_reporter,
readiness,
}
}
#[must_use]
pub fn extension_id(&self) -> ExtensionId {
self.name.clone()
}
pub fn signal_ready(&self) {
if let Some(sig) = &self.readiness {
sig.ready();
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ExtensionVariant {
Local,
Shared,
}
impl ExtensionVariant {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
ExtensionVariant::Local => "local",
ExtensionVariant::Shared => "shared",
}
}
}
pub enum ExtensionLifecycle<E, R> {
Active {
extension: E,
control_sender: Sender<ExtensionControlMsg>,
control_receiver: R,
shutdown_sender: Option<tokio::sync::oneshot::Sender<ShutdownPayload>>,
shutdown_receiver: Option<tokio::sync::oneshot::Receiver<ShutdownPayload>>,
readiness_probe: Option<super::readiness::ReadinessProbe>,
readiness_signaller: Option<super::readiness::ReadinessSignaller>,
},
Passive,
}
pub enum ExtensionWrapper {
Local {
name: ExtensionId,
user_config: Arc<ExtensionUserConfig>,
runtime_config: ExtensionConfig,
telemetry: Option<EntityTelemetryGuard>,
lifecycle: ExtensionLifecycle<
std::rc::Rc<dyn local_ext::Extension>,
LocalReceiver<ExtensionControlMsg>,
>,
instance_factory: LocalInstanceFactory,
},
Shared {
name: ExtensionId,
user_config: Arc<ExtensionUserConfig>,
runtime_config: ExtensionConfig,
telemetry: Option<EntityTelemetryGuard>,
lifecycle:
ExtensionLifecycle<Box<dyn shared_ext::Extension>, SharedReceiver<ExtensionControlMsg>>,
instance_factory: SharedInstanceFactory,
},
}
pub struct ExtensionBundle {
local: Option<ExtensionWrapper>,
shared: Option<ExtensionWrapper>,
}
impl ExtensionWrapper {
#[must_use]
pub fn name(&self) -> ExtensionId {
match self {
ExtensionWrapper::Local { name, .. } | ExtensionWrapper::Shared { name, .. } => {
name.clone()
}
}
}
#[must_use]
pub fn user_config(&self) -> Arc<ExtensionUserConfig> {
match self {
ExtensionWrapper::Local { user_config, .. }
| ExtensionWrapper::Shared { user_config, .. } => user_config.clone(),
}
}
#[must_use]
pub fn variant(&self) -> ExtensionVariant {
match self {
ExtensionWrapper::Local { .. } => ExtensionVariant::Local,
ExtensionWrapper::Shared { .. } => ExtensionVariant::Shared,
}
}
#[must_use]
pub fn is_passive(&self) -> bool {
match self {
ExtensionWrapper::Local { lifecycle, .. } => {
matches!(lifecycle, ExtensionLifecycle::Passive)
}
ExtensionWrapper::Shared { lifecycle, .. } => {
matches!(lifecycle, ExtensionLifecycle::Passive)
}
}
}
#[must_use]
pub fn shared_instance_factory(&self) -> Option<&SharedInstanceFactory> {
match self {
ExtensionWrapper::Shared {
instance_factory, ..
} => Some(instance_factory),
_ => None,
}
}
#[must_use]
pub fn local_instance_factory(&self) -> Option<&LocalInstanceFactory> {
match self {
ExtensionWrapper::Local {
instance_factory, ..
} => Some(instance_factory),
_ => None,
}
}
pub(crate) fn with_entity_telemetry_guard(mut self, guard: EntityTelemetryGuard) -> Self {
match &mut self {
ExtensionWrapper::Local { telemetry, .. }
| ExtensionWrapper::Shared { telemetry, .. } => *telemetry = Some(guard),
}
self
}
pub(crate) fn take_telemetry_guard(&mut self) -> Option<EntityTelemetryGuard> {
match self {
ExtensionWrapper::Local { telemetry, .. }
| ExtensionWrapper::Shared { telemetry, .. } => telemetry.take(),
}
}
pub(crate) fn with_control_channel_metrics(
self,
entity_handle: &EntityTelemetryHandle,
ext_ctx: &ExtensionContext,
channel_metrics: &mut ChannelMetricsRegistry,
channel_metrics_enabled: bool,
) -> Self {
match self {
ExtensionWrapper::Local {
name,
user_config,
runtime_config,
telemetry,
lifecycle,
instance_factory,
} => {
let lifecycle = match lifecycle {
ExtensionLifecycle::Passive => ExtensionLifecycle::Passive,
ExtensionLifecycle::Active {
extension,
control_sender,
control_receiver,
shutdown_sender,
shutdown_receiver,
readiness_probe,
readiness_signaller,
} => {
let capacity = runtime_config.control_channel.capacity as u64;
let (local_sender, local_receiver) = match control_sender {
Sender::Local(s) => (s, control_receiver),
_ => unreachable!("Local variant always has local sender"),
};
let (s, r) = wrap_extension_control_channel_metrics::<
LocalMode,
ExtensionControlMsg,
>(
name.clone(),
ExtensionVariant::Local,
entity_handle,
ext_ctx,
channel_metrics,
channel_metrics_enabled,
capacity,
local_sender,
local_receiver,
);
ExtensionLifecycle::Active {
extension,
control_sender: Sender::Local(s),
control_receiver: r,
shutdown_sender,
shutdown_receiver,
readiness_probe,
readiness_signaller,
}
}
};
ExtensionWrapper::Local {
name,
user_config,
runtime_config,
telemetry,
lifecycle,
instance_factory,
}
}
ExtensionWrapper::Shared {
name,
user_config,
runtime_config,
telemetry,
lifecycle,
instance_factory,
} => {
let lifecycle = match lifecycle {
ExtensionLifecycle::Passive => ExtensionLifecycle::Passive,
ExtensionLifecycle::Active {
extension,
control_sender,
control_receiver,
shutdown_sender,
shutdown_receiver,
readiness_probe,
readiness_signaller,
} => {
let capacity = runtime_config.control_channel.capacity as u64;
let shared_sender = match control_sender {
Sender::Shared(s) => s,
_ => unreachable!("Shared variant always has shared sender"),
};
let (s, r) = wrap_extension_control_channel_metrics::<
SharedMode,
ExtensionControlMsg,
>(
name.clone(),
ExtensionVariant::Shared,
entity_handle,
ext_ctx,
channel_metrics,
channel_metrics_enabled,
capacity,
shared_sender,
control_receiver,
);
ExtensionLifecycle::Active {
extension,
control_sender: Sender::Shared(s),
control_receiver: r,
shutdown_sender,
shutdown_receiver,
readiness_probe,
readiness_signaller,
}
}
};
ExtensionWrapper::Shared {
name,
user_config,
runtime_config,
telemetry,
lifecycle,
instance_factory,
}
}
}
}
#[allow(dead_code)]
pub(crate) fn extension_control_sender(
&self,
) -> Option<crate::control::ExtensionControlSender> {
let sender = match self {
ExtensionWrapper::Local {
lifecycle: ExtensionLifecycle::Active { control_sender, .. },
..
}
| ExtensionWrapper::Shared {
lifecycle: ExtensionLifecycle::Active { control_sender, .. },
..
} => control_sender,
_ => return None,
};
Some(crate::control::ExtensionControlSender {
sender: sender.clone(),
})
}
pub(crate) fn take_shutdown_sender(
&mut self,
) -> Option<crate::control::ExtensionShutdownChannel> {
let (name, sender_slot) = match self {
ExtensionWrapper::Local {
name,
lifecycle:
ExtensionLifecycle::Active {
shutdown_sender, ..
},
..
}
| ExtensionWrapper::Shared {
name,
lifecycle:
ExtensionLifecycle::Active {
shutdown_sender, ..
},
..
} => (name, shutdown_sender),
_ => return None,
};
let sender = sender_slot.take()?;
Some(crate::control::ExtensionShutdownChannel {
name: name.clone(),
sender,
})
}
pub(crate) fn take_readiness_probe(&mut self) -> Option<super::readiness::ReadinessProbe> {
let probe_slot = match self {
ExtensionWrapper::Local {
lifecycle:
ExtensionLifecycle::Active {
readiness_probe, ..
},
..
}
| ExtensionWrapper::Shared {
lifecycle:
ExtensionLifecycle::Active {
readiness_probe, ..
},
..
} => readiness_probe,
_ => return None,
};
probe_slot.take()
}
#[cfg(test)]
pub(crate) fn has_readiness_signaller(&self) -> bool {
match self {
ExtensionWrapper::Local {
lifecycle:
ExtensionLifecycle::Active {
readiness_signaller,
..
},
..
}
| ExtensionWrapper::Shared {
lifecycle:
ExtensionLifecycle::Active {
readiness_signaller,
..
},
..
} => readiness_signaller.is_some(),
_ => false,
}
}
pub(crate) async fn start(
self,
metrics_reporter: MetricsReporter,
) -> Result<TerminalState, Error> {
match self {
ExtensionWrapper::Local {
name,
lifecycle:
ExtensionLifecycle::Active {
extension,
control_receiver,
shutdown_receiver,
readiness_signaller,
..
},
..
} => {
otel_debug!("extension.start.local", name = name.as_ref());
let effect_handler =
EffectHandler::new(name, metrics_reporter, readiness_signaller);
let shutdown_rx = shutdown_receiver.expect(
"shutdown_receiver must be present when an active extension is started",
);
extension
.start(
local_ext::ControlChannel::new(control_receiver, shutdown_rx),
effect_handler,
)
.await
}
ExtensionWrapper::Shared {
name,
lifecycle:
ExtensionLifecycle::Active {
extension,
control_receiver,
shutdown_receiver,
readiness_signaller,
..
},
..
} => {
otel_debug!("extension.start.shared", name = name.as_ref());
let effect_handler =
EffectHandler::new(name, metrics_reporter, readiness_signaller);
let shutdown_rx = shutdown_receiver.expect(
"shutdown_receiver must be present when an active extension is started",
);
extension
.start(
shared_ext::ControlChannel::new(control_receiver, shutdown_rx),
effect_handler,
)
.await
}
_ => Err(Error::InternalError {
message: "start() called on passive extension".into(),
}),
}
}
}
impl ExtensionBundle {
pub(super) fn from_parts(
local: Option<ExtensionWrapper>,
shared: Option<ExtensionWrapper>,
) -> Self {
ExtensionBundle { local, shared }
}
#[must_use]
pub fn local(&self) -> Option<&ExtensionWrapper> {
self.local.as_ref()
}
#[must_use]
pub fn shared(&self) -> Option<&ExtensionWrapper> {
self.shared.as_ref()
}
pub fn take_local(&mut self) -> Option<ExtensionWrapper> {
self.local.take()
}
pub fn take_shared(&mut self) -> Option<ExtensionWrapper> {
self.shared.take()
}
pub fn iter(&self) -> impl Iterator<Item = &ExtensionWrapper> {
self.local.iter().chain(self.shared.iter())
}
pub fn register_into(
&self,
capabilities: Option<&crate::capability::ExtensionCapabilities>,
registry: &mut crate::capability::registry::CapabilityRegistry,
) -> Result<(), crate::capability::registry::Error> {
let Some(capabilities) = capabilities else {
return Ok(());
};
let bundle_name = self
.local
.as_ref()
.or(self.shared.as_ref())
.map(|w| w.name().to_string())
.unwrap_or_else(|| "<empty bundle>".to_string());
if !capabilities.shared.is_empty() && self.shared.is_none() {
return Err(crate::capability::registry::Error::InternalError {
message: format!(
"extension '{bundle_name}': extension_capabilities! advertises shared \
capabilities {caps:?} but the ExtensionBundle has no shared variant \
- either add `.shared(...)` to the builder chain or remove the \
capabilities from the macro list",
caps = capabilities.shared,
),
});
}
if !capabilities.local.is_empty() && self.local.is_none() {
return Err(crate::capability::registry::Error::InternalError {
message: format!(
"extension '{bundle_name}': extension_capabilities! advertises local \
capabilities {caps:?} but the ExtensionBundle has no local variant \
- either add `.local(...)` to the builder chain or remove the \
capabilities from the macro list",
caps = capabilities.local,
),
});
}
if let Some(shared) = self.shared.as_ref()
&& let Some(factory) = shared.shared_instance_factory()
{
(capabilities.register_shared)(shared.name(), factory.clone(), registry)?;
}
if let Some(local) = self.local.as_ref()
&& let Some(factory) = local.local_instance_factory()
{
(capabilities.register_local)(local.name(), factory.clone(), registry)?;
}
Ok(())
}
pub(crate) fn wire_telemetry(
&mut self,
extension_id: ExtensionId,
ext_ctx: &ExtensionContext,
channel_metrics: &mut ChannelMetricsRegistry,
channel_metrics_enabled: bool,
) -> ExtensionEntityKeys {
let mut keys = ExtensionEntityKeys::default();
if let Some(w) = self.local.take() {
let entity_key =
ext_ctx.register_extension_entity(extension_id.clone(), ExtensionVariant::Local);
let handle = EntityTelemetryHandle::new(ext_ctx.metrics_registry(), entity_key);
keys.local = Some(entity_key);
let w = w.with_control_channel_metrics(
&handle,
ext_ctx,
channel_metrics,
channel_metrics_enabled,
);
self.local = Some(w.with_entity_telemetry_guard(EntityTelemetryGuard::new(handle)));
}
if let Some(w) = self.shared.take() {
let entity_key =
ext_ctx.register_extension_entity(extension_id, ExtensionVariant::Shared);
let handle = EntityTelemetryHandle::new(ext_ctx.metrics_registry(), entity_key);
keys.shared = Some(entity_key);
let w = w.with_control_channel_metrics(
&handle,
ext_ctx,
channel_metrics,
channel_metrics_enabled,
);
self.shared = Some(w.with_entity_telemetry_guard(EntityTelemetryGuard::new(handle)));
}
keys
}
}
#[derive(Debug, Default, Clone, Copy)]
pub(crate) struct ExtensionEntityKeys {
pub local: Option<otel_arrow_dfe_telemetry::registry::EntityKey>,
pub shared: Option<otel_arrow_dfe_telemetry::registry::EntityKey>,
}