use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::Duration;
use tokio::task::JoinSet;
use tokio_util::sync::CancellationToken;
use crate::batching::models::BufferGauge;
use crate::config::main::stream_settings::stream_settings;
use crate::config::models::{Config, OversizedPolicy};
use crate::delivery::main::build_delivery::build_delivery;
use crate::delivery::models::{Delivery, DeliveryInputs};
use crate::exporter::_helpers::stream_task::{StreamInputs, run_stream};
use crate::exporter::models::Clients;
use crate::kafka::main::list_topics::list_topics;
use crate::kafka::main::prefetch_kbytes::prefetch_kbytes;
use crate::metrics::models::Metrics;
use crate::naming::main::resolve_streams::resolve_streams;
use crate::naming::main::shard_of::shard_of;
use crate::naming::models::Stream;
use crate::planning::main::apply_plan::apply_plan;
use crate::planning::main::build_plan::build_plan;
use crate::planning::main::load_path::load_path;
use crate::planning::main::read_existing::read_existing;
const INCOMPATIBLE_RECHECK: Duration = Duration::from_secs(3600);
const REPLACE_WAIT: Duration = Duration::from_secs(120);
struct Running {
topics: Vec<String>,
cancel: CancellationToken,
finished: CancellationToken,
}
pub(crate) struct Supervisor {
pub(crate) config: Arc<Config>,
pub(crate) clients: Clients,
pub(crate) metrics: Arc<Metrics>,
pub(crate) gauge: BufferGauge,
running: BTreeMap<String, Running>,
tasks: JoinSet<(String, Result<(), String>)>,
stream_count: usize,
incompatible: BTreeMap<String, std::time::Instant>,
}
impl Supervisor {
pub(crate) fn new(
config: Arc<Config>,
clients: Clients,
metrics: Arc<Metrics>,
gauge: BufferGauge,
) -> Self {
Self {
config,
clients,
metrics,
gauge,
running: BTreeMap::new(),
tasks: JoinSet::new(),
stream_count: 0,
incompatible: BTreeMap::new(),
}
}
pub(crate) async fn run(&mut self, cancel: &CancellationToken) -> Result<Vec<String>, String> {
self.discover(cancel).await?;
let mut discovery = tokio::time::interval(self.config.source.discovery_interval);
discovery.tick().await;
let outcome = loop {
tokio::select! {
() = cancel.cancelled() => break Ok(()),
_ = discovery.tick() => {
if let Err(error) = self.discover(cancel).await {
tracing::warn!(%error, "topic discovery failed; keeping the current streams");
}
}
Some(joined) = self.tasks.join_next() => {
if let Some(failure) = self.on_task_end(joined, cancel) {
break Err(failure);
}
}
}
};
self.stop_all().await;
outcome.map(|()| self.running.keys().cloned().collect())
}
fn on_task_end(
&mut self,
joined: Result<(String, Result<(), String>), tokio::task::JoinError>,
cancel: &CancellationToken,
) -> Option<String> {
match joined {
Ok((name, Err(error))) => Some(format!("stream {name} failed: {error}")),
Err(error) => Some(format!("a stream task panicked: {error}")),
Ok((name, Ok(()))) if !cancel.is_cancelled() => {
let replaced = self
.running
.get(&name)
.is_some_and(|running| !running.cancel.is_cancelled());
match replaced {
true => None,
false => {
tracing::debug!(stream = %name, "previous task for a replaced stream finished");
None
}
}
}
Ok(_) => None,
}
}
async fn discover(&mut self, cancel: &CancellationToken) -> Result<(), String> {
let topics = list_topics(&self.config).await?;
let source = &self.config.source;
let streams: Vec<Stream> = resolve_streams(&self.config, &topics)
.map_err(|problems| problems.join("; "))?
.into_iter()
.filter(|stream| shard_of(&stream.table, source.shards) == source.shard)
.collect();
self.stream_count = streams.len();
let changed: Vec<Stream> = streams
.into_iter()
.filter(|stream| {
self.running
.get(&stream.name)
.is_none_or(|running| running.topics != stream.topics)
})
.filter(|stream| {
self.incompatible
.get(&stream.name)
.is_none_or(|checked| checked.elapsed() >= INCOMPATIBLE_RECHECK)
})
.collect();
if changed.is_empty() {
return Ok(());
}
let changed = self.apply(changed).await?;
if changed.is_empty() {
return Ok(());
}
let wave = match self.config.source.startup_batch {
0 => changed.len(),
size => size,
};
for (index, stream) in changed.into_iter().enumerate() {
if index > 0 && index % wave == 0 {
tokio::select! {
() = cancel.cancelled() => return Ok(()),
() = tokio::time::sleep(self.config.source.startup_interval) => {}
}
}
self.start(stream, cancel).await?;
}
Ok(())
}
async fn apply(&mut self, streams: Vec<Stream>) -> Result<Vec<Stream>, String> {
let existing = read_existing(&self.clients.warehouse, &self.config)
.await
.map_err(|error| format!("cannot read existing objects: {error}"))?;
let mut plan = build_plan(
&self.config,
&streams,
&existing,
self.clients.warehouse.uses_key_pair(),
);
let incompatible = &mut self.incompatible;
plan.streams.retain(|stream_plan| {
let problems: Vec<String> = stream_plan
.objects
.iter()
.flat_map(|object| object.problems())
.collect();
let name = &stream_plan.stream.name;
match problems.is_empty() {
true => {
incompatible.remove(name);
}
false => {
tracing::error!(stream = %name, ?problems, "existing objects do not fit; not exporting this stream, checking again in an hour");
incompatible.insert(name.clone(), std::time::Instant::now());
}
}
problems.is_empty()
});
let created = apply_plan(&self.clients.warehouse, &plan)
.await
.map_err(|error| error.to_string())?;
for object in created {
tracing::info!(kind = %object.kind, name = %object.name, "created");
}
Ok(plan
.streams
.into_iter()
.map(|stream_plan| stream_plan.stream)
.collect())
}
async fn start(&mut self, stream: Stream, cancel: &CancellationToken) -> Result<(), String> {
if let Some(previous) = self.running.remove(&stream.name) {
tracing::info!(stream = %stream.name, topics = ?stream.topics, "topics changed; restarting the stream");
previous.cancel.cancel();
let stopped = tokio::time::timeout(REPLACE_WAIT, previous.finished.cancelled()).await;
if stopped.is_err() {
return Err(format!(
"stream {} did not stop within {REPLACE_WAIT:?}",
stream.name
));
}
}
let first_topic = stream.topics.first().cloned().unwrap_or_default();
let settings = stream_settings(&self.config, &first_topic);
let delivery = self.delivery(&stream, settings.oversized)?;
let child = cancel.child_token();
let inputs = StreamInputs {
config: self.config.clone(),
stream: stream.clone(),
settings,
delivery,
metrics: self.metrics.clone(),
gauge: self.gauge.clone(),
prefetch_kbytes: prefetch_kbytes(
self.config.source.prefetch_budget.0,
self.stream_count,
),
};
let name = stream.name.clone();
let token = child.clone();
let finished = CancellationToken::new();
let done = finished.clone().drop_guard();
self.tasks.spawn(async move {
let outcome = run_stream(inputs, token).await;
drop(done);
(name, outcome)
});
self.running.insert(
stream.name.clone(),
Running {
topics: stream.topics,
cancel: child,
finished,
},
);
Ok(())
}
fn delivery(&self, stream: &Stream, oversized: OversizedPolicy) -> Result<Delivery, String> {
let inputs = DeliveryInputs {
http: self.clients.http.clone(),
in_flight: self.clients.in_flight.clone(),
warehouse: self.clients.warehouse.clone(),
config: self.config.clone(),
path: load_path(&self.config, self.clients.warehouse.uses_key_pair()),
oversized,
};
build_delivery(&inputs, stream).map_err(|error| error.to_string())
}
async fn stop_all(&mut self) {
for running in self.running.values() {
running.cancel.cancel();
}
let wait = tokio::time::timeout(Duration::from_secs(120), async {
while let Some(joined) = self.tasks.join_next().await {
if let Ok((name, Err(error))) = joined {
tracing::warn!(stream = %name, %error, "stream ended with an error during shutdown");
}
}
});
if wait.await.is_err() {
tracing::warn!("streams did not stop within two minutes");
self.tasks.abort_all();
}
}
}