use {
super::{
super::{
Functions,
status::{CandidateInfo, CandidatesMap, HandlerConditions, When},
},
Caller,
builder::CallerConfig,
},
crate::{
discovery::{Catalog, Discovery, rtt::PeerInfo},
network::LocalNode,
primitives::{Datum, ShortFmtExt},
},
core::marker::PhantomData,
std::sync::Arc,
tokio::{sync::watch, task::JoinSet},
tokio_util::sync::CancellationToken,
};
pub(super) struct CallerWorker {
config: Arc<CallerConfig>,
local: LocalNode,
discovery: Discovery,
cancel: CancellationToken,
candidates: watch::Sender<CandidatesMap>,
online: watch::Sender<bool>,
online_when: HandlerConditions,
rtt_probes: JoinSet<()>,
}
impl CallerWorker {
pub fn spawn<Req: Datum, Res: Datum, E: Datum>(
config: CallerConfig,
functions: &Functions,
) -> Caller<Req, Res, E> {
let config = Arc::new(config);
let local = functions.local.clone();
let cancel = local.termination().child_token();
let candidates = watch::Sender::new(CandidatesMap::new());
let online = watch::Sender::new(false);
let when = When::new(candidates.subscribe(), online.subscribe());
let online_when = (config.online_when)(when.available());
online.send_replace(online_when.is_condition_met());
let metrics_labels = [
("function", config.function_id.short().to_string()),
("network", local.network_id().short().to_string()),
];
let worker = Self {
local: local.clone(),
config: Arc::clone(&config),
discovery: functions.discovery.clone(),
cancel: cancel.clone(),
candidates: candidates.clone(),
online: online.clone(),
online_when,
rtt_probes: JoinSet::new(),
};
tokio::spawn(worker.run());
Caller {
config,
local,
discovery: functions.discovery.clone(),
candidates: candidates.subscribe(),
status: When::new(candidates.subscribe(), online.subscribe()),
metrics_labels,
_abort: cancel.drop_guard(),
_marker: PhantomData,
}
}
async fn run(mut self) {
let mut catalog = self.discovery.catalog_watch();
catalog.mark_changed();
loop {
tokio::select! {
() = self.cancel.cancelled() => {
break;
}
() = &mut self.online_when => {
self.on_online();
}
_ = catalog.changed() => {
let snapshot = catalog.borrow_and_update().clone();
self.on_catalog_update(&snapshot);
}
Some(_) = self.rtt_probes.join_next() => {}
}
}
}
fn on_catalog_update(&mut self, latest: &Catalog) {
let mut eligible = CandidatesMap::new();
let handlers = latest
.peers()
.filter(|peer| peer.functions().contains(&self.config.function_id));
for handler in handlers {
if self.discovery.rtt_tracker().get(handler.id()).is_none() {
let local = self.local.clone();
let discovery = self.discovery.clone();
let addr = handler.address().clone();
self.rtt_probes.spawn(async move {
if let Ok((entry, rtt)) = local.ping(addr, None).await
&& let Some(rtt) = rtt
{
discovery.rtt_tracker().record_sample(*entry.id(), rtt);
discovery.feed(entry);
}
});
continue;
}
let info = PeerInfo::from_tracker(handler, self.discovery.rtt_tracker());
if !(self.config.require)(&info) {
tracing::debug!(
function_id = %self.config.function_id.short(),
handler_id = %handler.id().short(),
network = %handler.network_id().short(),
"skipping ineligible handler"
);
continue;
}
if handler.validate_tickets(&self.config.handler_auth).is_err() {
tracing::debug!(
function_id = %self.config.function_id.short(),
handler_id = %handler.id().short(),
network = %handler.network_id().short(),
"skipping unauthorized handler"
);
continue;
}
let rtt = info.rtt();
eligible.insert(*handler.id(), CandidateInfo {
entry: handler.clone(),
rtt,
});
}
self.candidates.send_replace(eligible);
if !self.online_when.is_condition_met() {
tracing::trace!(
function_id = %self.config.function_id.short(),
handlers = %self.candidates.borrow().len(),
"caller is offline",
);
self.online.send_replace(false);
}
}
fn on_online(&self) {
tracing::trace!(
function_id = %self.config.function_id.short(),
handlers = %self.candidates.borrow().len(),
"caller is online",
);
self.online.send_if_modified(|status| {
if *status {
false
} else {
*status = true;
true
}
});
}
}