use std::collections::VecDeque;
use std::io::Write;
use std::time::{Duration, Instant};
use astrs_data::builder::{BinaryBuilder, BooleanBuilder, Float64Builder};
use astrs_data::ipc::decode_payload;
use astrs_data::record_batch::RecordBatch;
use astrs_data::urn::layout_of;
use astrs_data::{DataType, IntoArrayRef};
use astrs_node_api::NodeBuilder;
use astrs_wire::{
ControlReply, ControlRequest, DataFrame, DataflowId, FrameKind, Metadata, NodeInfo, PortRef,
TopicQuery, TypeUrn, WireDecode,
};
use crate::command::client::{
Client, DataflowRef, Endpoint, new_subscription_id, reply_name, runtime,
};
use crate::command::param::parse_value as parse_parameter_value;
use crate::command::signals::Signals;
use crate::error::CliError;
#[derive(Debug, Clone)]
pub struct TopicStreamArgs {
pub topic: String,
pub dataflow: Option<String>,
pub json: bool,
pub count: Option<u32>,
}
pub fn parse_port(text: &str) -> Result<PortRef, CliError> {
text.parse()
.map_err(|error: astrs_wire::IdError| CliError::BadArgument {
flag: "topic",
value: text.to_owned(),
reason: error.to_string(),
})
}
async fn resolve_dataflow(
client: &mut Client,
dataflow: Option<&str>,
) -> Result<DataflowId, CliError> {
match dataflow {
Some(text) => client.resolve(&DataflowRef::parse(text)).await,
None => client.sole_dataflow(true).await,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct StreamReport {
pub received: usize,
}
pub fn echo(
out: &mut dyn Write,
endpoint: &Endpoint,
args: &TopicStreamArgs,
) -> Result<StreamReport, CliError> {
let port = parse_port(&args.topic)?;
let runtime = runtime()?;
runtime.block_on(async move {
let mut client = Client::connect(endpoint).await?;
let dataflow = resolve_dataflow(&mut client, args.dataflow.as_deref()).await?;
let subscription = new_subscription_id();
client
.request_ok(
"topic echo",
&ControlRequest::TopicSubscribe {
dataflow,
port: port.clone(),
query: TopicQuery::new(),
subscription,
},
)
.await?;
let mut signals = Signals::install();
let mut received = 0usize;
let mut stopped_early = false;
loop {
tokio::select! {
frame = client.next_frame() => match frame? {
Some(frame) => {
if let Some(data) = decode_data_frame(&frame) {
print_echo(out, &data, args.json);
received += 1;
if args.count.is_some_and(|limit| received >= limit as usize) {
stopped_early = true;
break;
}
}
}
None => break,
},
() = signals.next() => {
stopped_early = true;
break;
}
}
}
if stopped_early {
let _ = client
.request_ok(
"topic echo",
&ControlRequest::TopicUnsubscribe { subscription },
)
.await;
}
Ok(StreamReport { received })
})
}
pub fn hz(
out: &mut dyn Write,
endpoint: &Endpoint,
args: &TopicStreamArgs,
window: Duration,
) -> Result<StreamReport, CliError> {
let port = parse_port(&args.topic)?;
let runtime = runtime()?;
runtime.block_on(async move {
let mut client = Client::connect(endpoint).await?;
let dataflow = resolve_dataflow(&mut client, args.dataflow.as_deref()).await?;
let subscription = new_subscription_id();
client
.request_ok(
"topic hz",
&ControlRequest::TopicSubscribe {
dataflow,
port: port.clone(),
query: TopicQuery::new(),
subscription,
},
)
.await?;
let mut rate = HzWindow::new(window);
let mut signals = Signals::install();
let mut received = 0usize;
let mut stopped_early = false;
loop {
tokio::select! {
frame = client.next_frame() => match frame? {
Some(frame) => {
if frame.kind() == FrameKind::Data {
let now = Instant::now();
rate.record(now);
received += 1;
print_hz(out, &rate, now, args.json);
if args.count.is_some_and(|limit| received >= limit as usize) {
stopped_early = true;
break;
}
}
}
None => break,
},
() = signals.next() => {
stopped_early = true;
break;
}
}
}
if stopped_early {
let _ = client
.request_ok(
"topic hz",
&ControlRequest::TopicUnsubscribe { subscription },
)
.await;
}
Ok(StreamReport { received })
})
}
#[derive(Debug, Clone)]
pub struct HzWindow {
window: Duration,
arrivals: VecDeque<Instant>,
}
impl HzWindow {
#[must_use]
pub fn new(window: Duration) -> Self {
Self {
window,
arrivals: VecDeque::new(),
}
}
pub fn record(&mut self, now: Instant) {
self.arrivals.push_back(now);
self.evict(now);
}
fn evict(&mut self, now: Instant) {
while let Some(&oldest) = self.arrivals.front() {
if now.saturating_duration_since(oldest) > self.window {
self.arrivals.pop_front();
} else {
break;
}
}
}
#[must_use]
pub fn count(&self) -> usize {
self.arrivals.len()
}
#[must_use]
pub fn rate_per_sec(&self, now: Instant) -> Option<f64> {
let oldest = *self.arrivals.front()?;
let span = now
.saturating_duration_since(oldest)
.as_secs_f64()
.max(f64::EPSILON);
if self.arrivals.len() < 2 {
return None;
}
Some((self.arrivals.len() - 1) as f64 / span)
}
}
fn print_hz(out: &mut dyn Write, rate: &HzWindow, now: Instant, json: bool) {
let value = rate.rate_per_sec(now);
if json {
let _ = writeln!(
out,
"{}",
serde_json::json!({ "count": rate.count(), "hz": value })
);
} else {
match value {
Some(hz) => {
let _ = writeln!(out, "{hz:.2} Hz ({} in window)", rate.count());
}
None => {
let _ = writeln!(out, "… ({} in window)", rate.count());
}
}
}
let _ = out.flush();
}
fn decode_data_frame(frame: &astrs_wire::Frame) -> Option<DataFrame> {
if frame.kind() != FrameKind::Data {
return None;
}
DataFrame::decode_exact(frame.payload()).ok()
}
fn print_echo(out: &mut dyn Write, frame: &DataFrame, json: bool) {
let summary = summarize_payload(&frame.payload);
if json {
let _ = writeln!(
out,
"{}",
serde_json::json!({
"source": frame.source.to_string(),
"bytes": frame.payload.len(),
"summary": summary,
})
);
} else {
let _ = writeln!(out, "[{}] {summary}", frame.source);
}
let _ = out.flush();
}
#[must_use]
pub fn summarize_payload(payload: &[u8]) -> String {
match decode_payload(payload) {
Ok(batch) => summarize_batch(&batch),
Err(_) => format!(
"{} bytes (not decodable as Arrow IPC): {}",
payload.len(),
hex(payload)
),
}
}
fn summarize_batch(batch: &RecordBatch) -> String {
let columns: Vec<String> = batch
.schema()
.fields()
.iter()
.map(|field| format!("{}:{:?}", field.name(), field.data_type()))
.collect();
format!(
"{} row(s), {} column(s) [{}]",
batch.num_rows(),
batch.num_columns(),
columns.join(", ")
)
}
const HEX_PREVIEW_LIMIT: usize = 64;
#[must_use]
pub fn hex(bytes: &[u8]) -> String {
let shown = &bytes[..bytes.len().min(HEX_PREVIEW_LIMIT)];
let mut text = String::with_capacity(shown.len() * 2 + 4);
for byte in shown {
text.push_str(&format!("{byte:02x}"));
}
if bytes.len() > HEX_PREVIEW_LIMIT {
text.push_str("...");
}
text
}
pub fn info(
out: &mut dyn Write,
endpoint: &Endpoint,
topic: &str,
dataflow: Option<&str>,
manifest_path: Option<&std::path::Path>,
json: bool,
) -> Result<InfoReport, CliError> {
let port = parse_port(topic)?;
let runtime = runtime()?;
let report = runtime.block_on(async move {
let mut client = Client::connect(endpoint).await?;
let dataflow_id = resolve_dataflow(&mut client, dataflow).await?;
let node_info = match client
.request(
"topic info",
&ControlRequest::GetNodeInfo {
dataflow: dataflow_id,
node: port.node().clone(),
},
)
.await?
{
ControlReply::NodeInfo { nodes } => nodes.into_iter().next(),
other => {
return Err(CliError::UnexpectedReply {
request: "topic info",
reply: reply_name(&other),
});
}
};
let type_urn = node_info
.as_ref()
.and_then(|info: &NodeInfo| info.outputs.get(port.port()))
.and_then(|urn| urn.clone());
let observed = peek_one_frame(&mut client, dataflow_id, port.clone()).await;
let (observed_schema, schema_hash) = match observed {
Some((summary, hash)) => (Some(summary), hash),
None => (None, None),
};
Ok(InfoReport {
port: port.clone(),
type_urn,
observed_schema,
schema_hash,
subscribers: manifest_path
.map(|path| subscribers_of(path, &port))
.transpose()?
.unwrap_or_default(),
})
})?;
print_info(out, &report, json);
Ok(report)
}
const INFO_PEEK_TIMEOUT: Duration = Duration::from_millis(800);
fn summarize_with_hash(payload: &[u8]) -> (String, Option<astrs_data::SchemaHash>) {
match astrs_data::ipc::decode_payload(payload) {
Ok(batch) => (
summarize_batch(&batch),
Some(astrs_data::SchemaHash::of(batch.schema())),
),
Err(_) => (summarize_payload(payload), None),
}
}
async fn peek_one_frame(
client: &mut Client,
dataflow: DataflowId,
port: PortRef,
) -> Option<(String, Option<astrs_data::SchemaHash>)> {
let subscription = new_subscription_id();
client
.request_ok(
"topic info",
&ControlRequest::TopicSubscribe {
dataflow,
port,
query: TopicQuery::new(),
subscription,
},
)
.await
.ok()?;
let observed = tokio::time::timeout(INFO_PEEK_TIMEOUT, async {
loop {
match client.next_frame().await.ok()? {
Some(frame) => {
if let Some(data) = decode_data_frame(&frame) {
return Some(summarize_with_hash(&data.payload));
}
}
None => return None,
}
}
})
.await
.ok()
.flatten();
let _ = client
.request_ok(
"topic info",
&ControlRequest::TopicUnsubscribe { subscription },
)
.await;
observed
}
fn subscribers_of(path: &std::path::Path, port: &PortRef) -> Result<Vec<PortRef>, CliError> {
let content = std::fs::read_to_string(path).map_err(|err| CliError::io(path, err))?;
let manifest = astrs_manifest::Manifest::from_yaml_str(&content)?;
manifest.validate()?;
let base_dir = path
.parent()
.map(std::path::Path::to_path_buf)
.unwrap_or_default();
let loader = astrs_manifest::expand::FsModuleLoader;
let expanded = manifest.expand(&base_dir, &loader)?;
expanded.validate()?;
let wanted = port.to_string();
let mut subscribers = Vec::new();
for node in &expanded.nodes {
let Ok(consumer) = astrs_wire::NodeId::new(&node.id) else {
continue;
};
for (input_name, input) in &node.inputs {
if input.source != wanted {
continue;
}
if let Ok(input_id) = astrs_wire::DataId::new(input_name) {
subscribers.push(PortRef::new(consumer.clone(), input_id));
}
}
}
Ok(subscribers)
}
#[derive(Debug, Clone)]
pub struct InfoReport {
pub port: PortRef,
pub type_urn: Option<TypeUrn>,
pub observed_schema: Option<String>,
pub schema_hash: Option<astrs_data::SchemaHash>,
pub subscribers: Vec<PortRef>,
}
fn print_info(out: &mut dyn Write, report: &InfoReport, json: bool) {
if json {
let _ = writeln!(
out,
"{}",
serde_json::json!({
"port": report.port.to_string(),
"type_urn": report.type_urn.as_ref().map(ToString::to_string),
"plane": report.observed_schema.as_ref().map(|_| "daemon-mediated (tapped)"),
"observed_schema": report.observed_schema,
"schema_hash": report.schema_hash.as_ref().map(ToString::to_string),
"subscribers": report.subscribers.iter().map(ToString::to_string).collect::<Vec<_>>(),
})
);
} else {
let _ = writeln!(out, "{}", report.port);
let _ = writeln!(
out,
" type: {}",
report
.type_urn
.as_ref()
.map_or_else(|| "(untyped)".to_owned(), ToString::to_string)
);
match &report.observed_schema {
Some(schema) => {
let _ = writeln!(out, " plane: daemon-mediated (tapped)");
let _ = writeln!(out, " observed: {schema}");
match &report.schema_hash {
Some(hash) => {
let _ = writeln!(out, " schema_hash: {hash}");
}
None => {
let _ = writeln!(out, " schema_hash: (not decodable as Arrow IPC)");
}
}
}
None => {
let _ = writeln!(out, " plane: unknown (no frame observed)");
}
}
if report.subscribers.is_empty() {
let _ = writeln!(out, " subscribers: (pass --manifest to list them)");
} else {
let _ = writeln!(out, " subscribers:");
for subscriber in &report.subscribers {
let _ = writeln!(out, " {subscriber}");
}
}
}
let _ = out.flush();
}
pub fn publish(
out: &mut dyn Write,
endpoint: &Endpoint,
args: &PublishArgs,
) -> Result<PublishReport, CliError> {
let port = parse_port(&args.topic)?;
let runtime = runtime()?;
let lookup_port = port.clone();
let (dataflow, type_urn) = runtime.block_on(async move {
let mut client = Client::connect(endpoint).await?;
let dataflow = resolve_dataflow(&mut client, args.dataflow.as_deref()).await?;
let type_urn = match client
.request(
"topic pub",
&ControlRequest::GetNodeInfo {
dataflow,
node: lookup_port.node().clone(),
},
)
.await
{
Ok(ControlReply::NodeInfo { nodes }) => nodes
.into_iter()
.next()
.and_then(|info| info.outputs.get(lookup_port.port()).cloned())
.flatten(),
_ => None,
};
Ok::<_, CliError>((dataflow, type_urn))
})?;
let batch = encode_message(&args.message, type_urn.as_ref())?;
let repeat = args.rate.filter(|hz| *hz > 0.0);
let attach_port = port.clone();
let sent = runtime.block_on(async move {
let builder = NodeBuilder::new()
.dataflow(dataflow)
.node_id(attach_port.node().as_str())?
.dynamic(true)
.output(attach_port.port().as_str())?
.auth(endpoint.token.clone());
let (mut node, _events) = builder.connect_async().await?;
let mut output = node.raw_output(attach_port.port().as_str())?;
output.send_batch(&batch, Metadata::new(node.hlc_now()))?;
let mut sent = 1usize;
if let Some(hz) = repeat {
let mut signals = Signals::install();
let interval = Duration::from_secs_f64(1.0 / hz);
loop {
tokio::select! {
() = tokio::time::sleep(interval) => {
output.send_batch(&batch, Metadata::new(node.hlc_now()))?;
sent += 1;
}
() = signals.next() => break,
}
}
}
let _ = node.shutdown();
node.flush_outputs().await;
Ok::<_, CliError>(sent)
})?;
let report = PublishReport { port, sent };
if args.json {
let _ = writeln!(
out,
"{}",
serde_json::json!({ "port": report.port.to_string(), "sent": report.sent })
);
} else {
let _ = writeln!(
out,
"published {} message(s) on {}",
report.sent, report.port
);
}
let _ = out.flush();
Ok(report)
}
#[derive(Debug, Clone)]
pub struct PublishArgs {
pub topic: String,
pub message: String,
pub dataflow: Option<String>,
pub rate: Option<f64>,
pub json: bool,
}
#[derive(Debug, Clone)]
pub struct PublishReport {
pub port: PortRef,
pub sent: usize,
}
fn encode_message(message: &str, urn: Option<&TypeUrn>) -> Result<RecordBatch, CliError> {
if let Some(urn) = urn
&& let Ok(data_urn) = astrs_data::urn::TypeUrn::parse(urn.as_str())
&& let Ok(layout) = layout_of(&data_urn)
&& let Some(batch) = typed_batch(message, &layout)?
{
return Ok(batch);
}
let mut builder = BinaryBuilder::with_capacity(1, message.len());
builder.append_value(message.as_bytes());
Ok(RecordBatch::from_payload(builder.finish().into_array_ref()))
}
fn typed_batch(message: &str, layout: &DataType) -> Result<Option<RecordBatch>, CliError> {
use astrs_data::builder::{Int64Builder, StringBuilder};
use astrs_wire::Parameter;
let mismatch = || CliError::BadArgument {
flag: "message",
value: message.to_owned(),
reason: format!("the port's declared type needs a {layout:?}, which that value is not"),
};
let batch = match layout {
DataType::Bool => {
let Parameter::Bool(value) = parse_parameter_value(message)? else {
return Err(mismatch());
};
let mut builder = BooleanBuilder::with_capacity(1);
builder.append_value(value);
RecordBatch::from_payload(builder.finish().into_array_ref())
}
DataType::Float64 => {
let value = match parse_parameter_value(message)? {
Parameter::Float(value) => value,
#[allow(clippy::cast_precision_loss)]
Parameter::Integer(value) => value as f64,
_ => return Err(mismatch()),
};
let mut builder = Float64Builder::with_capacity(1);
builder.append_value(value);
RecordBatch::from_payload(builder.finish().into_array_ref())
}
DataType::Int64 => {
let Parameter::Integer(value) = parse_parameter_value(message)? else {
return Err(mismatch());
};
let mut builder = Int64Builder::with_capacity(1);
builder.append_value(value);
RecordBatch::from_payload(builder.finish().into_array_ref())
}
DataType::Utf8 => {
let Parameter::String(value) = parse_parameter_value(message)? else {
return Err(mismatch());
};
let mut builder = StringBuilder::with_capacity(1, value.len());
builder.append_value(&value);
RecordBatch::from_payload(builder.finish().into_array_ref())
}
_ => return Ok(None),
};
Ok(Some(batch))
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
#[test]
fn a_port_reference_parses_node_and_output() {
let port = parse_port("camera/frames").unwrap();
assert_eq!(port.node().as_str(), "camera");
assert_eq!(port.port().as_str(), "frames");
}
#[test]
fn a_port_reference_with_no_separator_is_refused() {
let error = parse_port("camera").unwrap_err();
assert!(matches!(error, CliError::BadArgument { flag: "topic", .. }));
}
#[test]
fn hz_window_is_empty_before_any_arrival() {
let window = HzWindow::new(Duration::from_secs(5));
let now = Instant::now();
assert_eq!(window.count(), 0);
assert_eq!(window.rate_per_sec(now), None);
}
#[test]
fn hz_window_reports_none_on_a_single_arrival() {
let mut window = HzWindow::new(Duration::from_secs(5));
let now = Instant::now();
window.record(now);
assert_eq!(window.count(), 1);
assert_eq!(window.rate_per_sec(now), None);
}
#[test]
fn hz_window_computes_the_rate_across_evenly_spaced_arrivals() {
let mut window = HzWindow::new(Duration::from_secs(10));
let start = Instant::now();
for step in 0..5u64 {
window.record(start + Duration::from_millis(step * 100));
}
let now = start + Duration::from_millis(400);
let hz = window.rate_per_sec(now).unwrap();
assert!((hz - 10.0).abs() < 1e-6, "{hz}");
}
#[test]
fn hz_window_evicts_arrivals_older_than_the_window() {
let mut window = HzWindow::new(Duration::from_millis(500));
let start = Instant::now();
window.record(start);
window.record(start + Duration::from_millis(100));
window.record(start + Duration::from_secs(2));
assert_eq!(window.count(), 1);
}
#[test]
fn a_valid_arrow_payload_summarizes_with_its_schema() {
use astrs_data::builder::Int64Builder;
use astrs_data::ipc::encode_payload;
let mut builder = Int64Builder::with_capacity(3);
builder.append_value(1);
builder.append_value(2);
builder.append_value(3);
let batch = RecordBatch::from_payload(builder.finish().into_array_ref());
let encoded = encode_payload(&batch).expect("a valid batch encodes");
let summary = summarize_payload(encoded.as_ref());
assert!(summary.contains("3 row(s)"), "{summary}");
assert!(summary.contains("1 column(s)"), "{summary}");
assert!(summary.contains("Int64"), "{summary}");
assert!(
!summary.contains("not decodable"),
"a real Arrow payload must not fall back to hex: {summary}"
);
}
#[test]
fn a_non_arrow_payload_summarizes_as_hex() {
let summary = summarize_payload(b"not arrow at all");
assert!(summary.contains("not decodable"), "{summary}");
assert!(summary.contains("6e6f7420"), "{summary}");
}
#[test]
fn a_valid_arrow_payload_also_carries_its_schema_hash() {
use astrs_data::builder::Int64Builder;
use astrs_data::ipc::encode_payload;
let mut builder = Int64Builder::with_capacity(1);
builder.append_value(1);
let batch = RecordBatch::from_payload(builder.finish().into_array_ref());
let encoded = encode_payload(&batch).expect("a valid batch encodes");
let (summary, hash) = summarize_with_hash(encoded.as_ref());
assert!(summary.contains("Int64"), "{summary}");
let hash = hash.expect("a decodable batch has a schema to hash");
assert_eq!(
hash,
astrs_data::SchemaHash::of(batch.schema()),
"`info` must hash exactly the schema blueprint §6.1 defines, \
not a summary of it"
);
}
#[test]
fn a_non_arrow_payload_has_no_schema_hash_but_still_summarizes() {
let (summary, hash) = summarize_with_hash(b"not arrow at all");
assert!(summary.contains("not decodable"), "{summary}");
assert!(hash.is_none(), "raw bytes have no schema to hash");
}
#[test]
fn hex_truncates_long_payloads_with_a_marker() {
let bytes = vec![0xAB; HEX_PREVIEW_LIMIT + 10];
let text = hex(&bytes);
assert!(text.ends_with("..."));
assert_eq!(text.len(), HEX_PREVIEW_LIMIT * 2 + 3);
}
#[test]
fn an_untyped_message_encodes_as_a_binary_column() {
let batch = encode_message("hello", None).unwrap();
assert_eq!(batch.num_rows(), 1);
assert_eq!(batch.schema().fields()[0].data_type(), &DataType::Binary);
}
#[test]
fn a_float_urn_encodes_a_plain_number_as_float64() {
let urn = TypeUrn::new("std/core/v1/Float64").unwrap();
let batch = encode_message("1.5", Some(&urn)).unwrap();
assert_eq!(batch.schema().fields()[0].data_type(), &DataType::Float64);
}
#[test]
fn a_bool_urn_rejects_a_non_boolean_value() {
let urn = TypeUrn::new("std/core/v1/Bool").unwrap();
let error = encode_message("\"left\"", Some(&urn)).unwrap_err();
assert!(matches!(
error,
CliError::BadArgument {
flag: "message",
..
}
));
}
#[test]
fn every_topic_verb_reports_a_dead_cluster_rather_than_hanging() {
let endpoint = Endpoint::new(
std::net::SocketAddr::from(([127, 0, 0, 1], 1)),
astrs_wire::AuthToken::ZERO,
);
let stream_args = TopicStreamArgs {
topic: "camera/frames".to_owned(),
dataflow: None,
json: false,
count: None,
};
let echo_error = echo(&mut Vec::new(), &endpoint, &stream_args).unwrap_err();
assert!(
matches!(echo_error, CliError::NoCluster { .. }),
"{echo_error}"
);
let hz_error = hz(
&mut Vec::new(),
&endpoint,
&stream_args,
Duration::from_secs(5),
)
.unwrap_err();
assert!(matches!(hz_error, CliError::NoCluster { .. }), "{hz_error}");
let info_error = info(
&mut Vec::new(),
&endpoint,
"camera/frames",
None,
None,
false,
)
.unwrap_err();
assert!(
matches!(info_error, CliError::NoCluster { .. }),
"{info_error}"
);
let publish_args = PublishArgs {
topic: "camera/frames".to_owned(),
message: "1".to_owned(),
dataflow: None,
rate: None,
json: false,
};
let publish_error = publish(&mut Vec::new(), &endpoint, &publish_args).unwrap_err();
assert!(
matches!(publish_error, CliError::NoCluster { .. }),
"{publish_error}"
);
}
}