use std::time::Duration;
use crate::{Error, Result};
use zenkey::grammar::with_base;
use zenkey::pattern::{PatternChunk, SubjectPattern};
use zenkey::qos::QosProfile;
use zenkey::schema::SchemaSet;
use zenkey::{Class, Declared, RateClass};
use crate::model::decode::SchemaStore;
use crate::model::registry::SliceSet;
use crate::report::{Fault, GenPlanEntry, GenReport};
use crate::tape::synth::Synth;
pub fn synthetic_marker(tool: &str, origin: &str, fault: Option<&str>) -> Vec<u8> {
let mut obj = serde_json::json!({
"synthetic": true,
"tool": tool,
"origin": origin,
});
if let Some(kind) = fault {
obj["fault"] = kind.into();
}
serde_json::to_vec(&obj).expect("the marker serializes")
}
impl Fault {
pub fn as_str(self) -> &'static str {
match self {
Fault::Truncate => "truncate",
Fault::WrongType => "wrong-type",
Fault::ExtraField => "extra-field",
Fault::UnregisteredKey => "unregistered-key",
Fault::WrongQos => "wrong-qos",
Fault::MissingEncoding => "missing-encoding",
Fault::Unstamped => "unstamped",
}
}
pub const ALL: [Fault; 7] = [
Fault::Truncate,
Fault::WrongType,
Fault::ExtraField,
Fault::UnregisteredKey,
Fault::WrongQos,
Fault::MissingEncoding,
Fault::Unstamped,
];
pub fn parse(s: &str) -> Result<Fault> {
Fault::ALL
.into_iter()
.find(|f| f.as_str() == s)
.ok_or_else(|| {
let known = Fault::ALL.map(Fault::as_str).join(", ");
Error::unaskable(
format!("--fault {s:?}"),
format!("is not a known fault kind — known kinds: {known}"),
)
})
}
fn perturb_key(self, key: &str) -> String {
match self {
Fault::UnregisteredKey => format!("{key}/unregistered"),
_ => key.to_string(),
}
}
fn perturb_qos(self, declared: QosProfile) -> QosProfile {
match self {
Fault::WrongQos if declared == QosProfile::Sampled => QosProfile::Transition,
Fault::WrongQos => QosProfile::Sampled,
_ => declared,
}
}
fn drops_encoding(self) -> bool {
matches!(self, Fault::MissingEncoding)
}
fn drops_timestamp(self) -> bool {
matches!(self, Fault::Unstamped)
}
fn perturb_body(self, bytes: Vec<u8>) -> Vec<u8> {
match self {
Fault::Truncate => {
let n = bytes.len() / 2;
let mut out = bytes;
out.truncate(n);
out
}
Fault::WrongType => {
serde_json::to_vec(&serde_json::Value::String("fault:wrong-type".into()))
.expect("a string serializes")
}
Fault::ExtraField => match serde_json::from_slice::<serde_json::Value>(&bytes) {
Ok(serde_json::Value::Object(mut m)) => {
m.insert("_fault".into(), serde_json::Value::Bool(true));
serde_json::to_vec(&serde_json::Value::Object(m)).expect("object serializes")
}
Ok(other) => {
let wrapped = serde_json::json!({ "_orig": other, "_fault": true });
serde_json::to_vec(&wrapped).expect("object serializes")
}
Err(_) => {
let mut out = bytes;
out.extend_from_slice(b"_fault");
out
}
},
_ => bytes,
}
}
fn delta(self, valid: &GenPlanEntry) -> String {
match self {
Fault::Truncate => {
"payload truncated to half its encoded bytes — a partial frame".into()
}
Fault::WrongType => format!(
"body replaced with a JSON string where {} is declared",
valid.type_name
),
Fault::ExtraField => "an undeclared `_fault` field added to the body".into(),
Fault::UnregisteredKey => format!(
"key → {} (an unregistered subject; RFC 09 §5.1 O1: a fact to report)",
self.perturb_key(&valid.key)
),
Fault::WrongQos => format!(
"qos {} → {} (declared profile not honoured, RFC 04 §3)",
valid.qos,
self.perturb_qos(QosProfile::from_name(&valid.qos).unwrap_or(QosProfile::Sampled))
.name()
),
Fault::MissingEncoding => match &valid.encoding {
Some(e) => format!("wire encoding {e} omitted"),
None => "no wire encoding set (none was declared either)".into(),
},
Fault::Unstamped => "no HLC timestamp — state LWW cannot order it (RFC 04 §4)".into(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GenPattern {
Steady,
Jitter,
Burst,
Ramp,
}
#[derive(Debug, Clone)]
pub struct GenSpec {
pub origin: String,
pub producer: Option<String>,
pub subject: Option<String>,
pub vars: Vec<(String, String)>,
pub rate_hz: Option<f64>,
pub pattern: GenPattern,
pub duration: Duration,
pub seed: u64,
pub tool: String,
pub faults: Vec<Fault>,
}
fn synthetic_var(name: &str) -> String {
let clean: String = name
.chars()
.filter(|c| c.is_ascii_alphanumeric())
.flat_map(|c| c.to_lowercase())
.collect();
if clean.is_empty() {
"v1".into()
} else {
format!("{clean}1")
}
}
pub async fn build_plan(
fleet: Option<&crate::Fleet<'_>>,
store: &SchemaStore,
slices: &SliceSet,
base: &str,
schema_set: Option<&SchemaSet>,
spec: &GenSpec,
) -> Result<Vec<GenPlanEntry>> {
let session = fleet.map(crate::Fleet::session);
let mut plan = Vec::new();
for slice in slices.slices() {
if slice.service_origin.is_some() {
continue;
}
if let Some(p) = &spec.producer
&& &slice.name != p
{
continue;
}
for subject in &slice.subjects {
if let Some(filter) = &spec.subject
&& !subject.path.contains(filter.as_str())
{
continue;
}
let pattern = SubjectPattern::parse(&subject.path).map_err(|e| {
Error::unaskable(format!("{}/{}", slice.name, subject.path), e.to_string())
})?;
let mut tail: Vec<String> = Vec::new();
let mut synthetic_vars: Vec<String> = Vec::new();
let mut unique_tail_idx = None;
for chunk in pattern.chunks() {
match chunk {
PatternChunk::Literal(l) => tail.push(l.clone()),
PatternChunk::Var(name) | PatternChunk::Rest(name) => {
let value = spec
.vars
.iter()
.find(|(k, _)| k == name)
.map(|(_, v)| v.clone())
.unwrap_or_else(|| {
synthetic_vars.push(name.clone());
synthetic_var(name)
});
if subject.class.is(&Class::Events) {
unique_tail_idx = Some(tail.len());
}
tail.push(value);
}
}
}
let key = with_base(
base,
format!(
"v1/{}/{}/{}/{}",
spec.origin,
subject.class,
slice.name,
tail.join("/")
),
);
let base_chunks = if base.is_empty() {
0
} else {
base.split('/').count()
};
let unique_chunk = unique_tail_idx.map(|i| base_chunks + 4 + i);
let (qos, qos_source) = match subject.qos.as_ref().and_then(Declared::known) {
Some(q) => (*q, "declared"),
None => (QosProfile::Sampled, "default"),
};
let mut events_cap = None;
let mut note: Option<String> = None;
let rate_hz = match subject.class.known() {
Some(Class::Events) => {
let cap_h = subject
.rate
.as_ref()
.and_then(RateClass::cap_per_hour)
.unwrap_or(1);
let cap_run = ((f64::from(u32::try_from(cap_h.min(3600)).unwrap_or(3600))
* spec.duration.as_secs_f64())
/ 3600.0)
.floor()
.max(1.0) as u64;
events_cap = Some(cap_run.min(cap_h));
(events_cap.unwrap_or(1) as f64 / spec.duration.as_secs_f64()).min(1.0)
}
Some(Class::State) => match subject.ttl_s {
Some(ttl) if ttl > 0 => 2.0 / ttl as f64,
_ => 0.5,
},
_ => 1.0,
};
let rate_hz = spec.rate_hz.unwrap_or(rate_hz).clamp(0.001, 1000.0);
let mut body_source = "placeholder";
let mut schema = None;
if let Some(session) = session
&& let Some(s) = store
.schema_for(session, &slice.name, &subject.type_name)
.await
{
schema = Some(s);
body_source = "describe";
}
if schema.is_none()
&& let Some(set) = schema_set
&& let Some(s) = set.get(&subject.type_name)
{
schema = Some(s.clone());
body_source = "schema-set";
}
if schema.is_none() {
note = Some(format!(
"no schema for {} — sending a placeholder {{}} body, labelled",
subject.type_name
));
}
if !synthetic_vars.is_empty() {
let vars = synthetic_vars.join(", ");
note = Some(match note.take() {
Some(n) => format!("{n}; synthetic values for {{{vars}}}"),
None => format!("synthetic values for {{{vars}}} (override with --var)"),
});
}
let encoding =
crate::bus::body::encode_encoding(None, subject.encoding.as_ref(), schema.as_ref());
let valid = GenPlanEntry {
key,
class: subject.class.token().to_string(),
producer: slice.name.clone(),
type_name: subject.type_name.clone(),
qos: qos.name().to_string(),
qos_source,
rate_hz,
body_source,
encoding,
events_cap,
note,
fault: None,
fault_delta: None,
schema,
unique_chunk,
};
if spec.faults.is_empty() {
plan.push(valid);
continue;
}
for &fault in &spec.faults {
let mut variant = valid.clone();
variant.fault_delta = Some(fault.delta(&valid));
variant.key = fault.perturb_key(&valid.key);
variant.qos = fault
.perturb_qos(QosProfile::from_name(&valid.qos).unwrap_or(QosProfile::Sampled))
.name()
.to_string();
if fault.drops_encoding() {
variant.encoding = None;
}
variant.fault = Some(fault);
plan.push(variant);
}
}
}
Ok(plan)
}
#[derive(Debug)]
pub struct MockProducer {
pub keys: usize,
tasks: Vec<tokio::task::JoinHandle<()>>,
}
impl Drop for MockProducer {
fn drop(&mut self) {
for t in &self.tasks {
t.abort();
}
}
}
pub async fn serve_describe(
fleet: &crate::Fleet<'_>,
origin: &str,
slices: &SliceSet,
schema_set: Option<&SchemaSet>,
producer: Option<&str>,
) -> Result<MockProducer> {
let (session, base) = (fleet.session(), fleet.base());
let mut up = crate::bus::producer::BringUp::new(session);
let mut bodies: Vec<(Vec<u8>, &'static str)> = Vec::new();
for (slice, raw) in slices.entries() {
if slice.service_origin.is_some() {
continue;
}
if let Some(p) = producer
&& slice.name != p
{
continue;
}
if raw.is_empty() {
continue; }
let introspect = with_base(base, format!("v1/{origin}/@rpc/{}/introspect", slice.name));
up.serve(&introspect).await?;
bodies.push((raw.as_bytes().to_vec(), "text/plain"));
if let Some(set) = schema_set {
let describe = with_base(base, format!("v1/{origin}/@rpc/{}/describe", slice.name));
up.serve(&describe).await?;
bodies.push((set.to_json().into_bytes(), "application/json"));
}
}
let responders = up.without_alive();
let keys = responders.len();
let mut tasks = Vec::new();
for (responder, (body, encoding)) in responders.into_iter().zip(bodies) {
tasks.push(tokio::spawn(async move {
while let Some(query) = responder.next().await {
if let Err(e) = responder.reply(&query, body.clone(), Some(encoding)).await {
tracing::warn!(key = %responder.key(), "mock producer reply failed: {e}");
}
}
}));
}
Ok(MockProducer { keys, tasks })
}
pub async fn run_gen(
fleet: &crate::Fleet<'_>,
plan: &[GenPlanEntry],
spec: &GenSpec,
) -> Result<GenReport> {
let session = fleet.session();
let synth = Synth::new(spec.seed);
let deadline = tokio::time::Instant::now() + spec.duration;
let total_s = spec.duration.as_secs_f64();
let mut tasks: tokio::task::JoinSet<(usize, u64, u64, Vec<String>)> =
tokio::task::JoinSet::new();
for (i, entry) in plan.iter().enumerate() {
let entry = entry.clone();
let session = session.clone();
let marker = synthetic_marker(&spec.tool, &spec.origin, entry.fault.map(Fault::as_str));
let store_encoding = entry.encoding.clone();
let pattern = spec.pattern;
let seed = spec.seed;
tasks.spawn(async move {
let registry = zenkey::schema::decode::DecoderRegistry::new();
let started = tokio::time::Instant::now();
let mut sent = 0u64;
let mut refused = 0u64;
let mut first_errors: Vec<String> = Vec::new();
let record_err = |e: String, refused: &mut u64, errs: &mut Vec<String>| {
*refused += 1;
if errs.len() < 3 {
errs.push(e);
}
};
let publication = if entry.unique_chunk.is_none() {
match crate::bus::write::declare_publication(
&session,
&entry.key,
QosProfile::from_name(&entry.qos).unwrap_or(QosProfile::Sampled),
entry.encoding.as_deref(),
)
.await
{
Ok(p) => Some(p),
Err(e) => {
return (i, 0, 1, vec![format!("{}: declare: {e}", entry.key)]);
}
}
} else {
None
};
let base_interval = Duration::from_secs_f64(1.0 / entry.rate_hz);
let mut tick: u64 = 0;
let run_over = tokio::time::sleep_until(deadline);
tokio::pin!(run_over);
loop {
if let Some(cap) = entry.events_cap
&& sent >= cap
{
(&mut run_over).await;
break;
}
let bytes = match &entry.schema {
Some(schema) => match synth.instance(schema, tick) {
Some(value) => {
let wire = zenkey::schema::WireEncoding::from_encoding_str(
store_encoding.as_deref().unwrap_or("application/json"),
);
match registry.encode(schema, &value, &wire) {
Ok(b) => b,
Err(e) => {
record_err(
format!("{}: encode: {e}", entry.key),
&mut refused,
&mut first_errors,
);
tick += 1;
continue;
}
}
}
None => b"{}".to_vec(),
},
None => b"{}".to_vec(),
};
let bytes = match entry.fault {
Some(f) => f.perturb_body(bytes),
None => bytes,
};
let stamp = if entry.fault.map(Fault::drops_timestamp).unwrap_or(false) {
None
} else {
Some(session.new_timestamp())
};
let outcome = match &publication {
Some(p) => p.send_stamped(bytes, Some(marker.clone()), stamp).await,
None => {
let key = unique_key(&entry, seed, sent);
match crate::bus::write::declare_publication(
&session,
&key,
QosProfile::from_name(&entry.qos).unwrap_or(QosProfile::Sampled),
entry.encoding.as_deref(),
)
.await
{
Ok(p) => {
let r = p.send_stamped(bytes, Some(marker.clone()), stamp).await;
let _ = p.undeclare().await;
r
}
Err(e) => Err(e),
}
}
};
match outcome {
Ok(()) => sent += 1,
Err(e) => record_err(
format!("{}: send: {e}", entry.key),
&mut refused,
&mut first_errors,
),
}
tick += 1;
let interval = match pattern {
GenPattern::Steady => base_interval,
GenPattern::Jitter => {
let f = 0.7 + 0.6 * halton(seed ^ (i as u64) ^ tick);
base_interval.mul_f64(f)
}
GenPattern::Burst => {
let per_burst = entry.rate_hz.ceil().max(1.0) as u64;
if tick.is_multiple_of(per_burst) {
Duration::from_secs(1)
} else {
Duration::ZERO
}
}
GenPattern::Ramp => {
let progress = (started.elapsed().as_secs_f64() / total_s).clamp(0.05, 1.0);
base_interval.div_f64(progress)
}
};
tokio::select! {
_ = tokio::time::sleep(interval) => {}
() = &mut run_over => break,
}
if tokio::time::Instant::now() >= deadline {
break;
}
}
if let Some(p) = publication {
let _ = p.undeclare().await;
}
(i, sent, refused, first_errors)
});
}
let mut done: Vec<Option<(u64, u64, Vec<String>)>> = vec![None; plan.len()];
let mut failed: Option<Error> = None;
while let Some(joined) = tasks.join_next().await {
match joined {
Ok((i, s, r, errs)) => done[i] = Some((s, r, errs)),
Err(e) => {
failed = Some(Error::Internal(format!("a gen task did not join: {e}")));
break;
}
}
}
tasks.shutdown().await;
if let Some(e) = failed {
return Err(e);
}
let mut sent = 0u64;
let mut refused = 0u64;
let mut first_errors = Vec::new();
for (s, r, errs) in done.into_iter().flatten() {
sent += s;
refused += r;
for e in errs {
if first_errors.len() < 5 {
first_errors.push(e);
}
}
}
Ok(GenReport {
duration_s: spec.duration.as_secs_f64(),
entries: plan.len(),
sent,
refused,
first_errors,
})
}
fn unique_key(entry: &GenPlanEntry, seed: u64, n: u64) -> String {
let Some(idx) = entry.unique_chunk else {
return entry.key.clone();
};
let id = format!("{:012x}{:04x}", seed & 0xffff_ffff_ffff, n & 0xffff);
entry
.key
.split('/')
.enumerate()
.map(|(i, c)| if i == idx { id.as_str() } else { c })
.collect::<Vec<_>>()
.join("/")
}
fn halton(n: u64) -> f64 {
let mut f = 1.0;
let mut r = 0.0;
let mut i = n.wrapping_mul(2654435761) % 4096 + 1;
while i > 0 {
f /= 2.0;
r += f * (i % 2) as f64;
i /= 2;
}
r
}
#[cfg(test)]
mod tests {
use super::*;
const SLICES: &str = r#"
[registry]
version = "1.0"
app = "t"
convention = 1
[producer]
name = "demo"
[[subject]]
path = "health"
class = "state"
type = "Health"
qos = "transition"
ttl_s = 30
[[subject]]
path = "cpu/{core}/usage"
class = "telemetry"
type = "Point"
[[subject]]
path = "boom/{id}"
class = "events"
type = "Boom"
rate = "rare"
"#;
fn spec() -> GenSpec {
GenSpec {
origin: "h-abababababab".into(),
producer: None,
subject: None,
vars: vec![("core".into(), "cpu0".into())],
rate_hz: None,
pattern: GenPattern::Steady,
duration: Duration::from_secs(10),
seed: 42,
tool: "zenctl gen".into(),
faults: vec![],
}
}
async fn plan_for(base: &str) -> Vec<GenPlanEntry> {
let slices =
SliceSet::from_slices(vec![zenkey::parse_slice(SLICES).expect("fixture parses")]);
let store = SchemaStore::new(base, Duration::from_millis(100));
let set = SchemaSet::parse(
r#"{"schema_version":1,"app":"t","types":{
"Health":{"kind":"json-schema","hash":"","schema":{"type":"object",
"properties":{"ok":{"type":"boolean"}}}}}}"#,
)
.expect("set parses");
build_plan(None, &store, &slices, base, Some(&set), &spec())
.await
.expect("plan builds")
}
#[tokio::test]
async fn the_plan_resolves_declared_qos_rates_and_the_schema_ladder() {
let plan = plan_for("").await;
assert_eq!(plan.len(), 3);
let health = &plan[0];
assert_eq!(health.key, "v1/h-abababababab/state/demo/health");
assert_eq!(
(health.qos.as_str(), health.qos_source),
("transition", "declared")
);
assert!(
(health.rate_hz - 2.0 / 30.0).abs() < 1e-9,
"{}",
health.rate_hz
);
assert_eq!(health.body_source, "schema-set");
assert!(health.note.is_none());
let cpu = &plan[1];
assert_eq!(cpu.key, "v1/h-abababababab/telemetry/demo/cpu/cpu0/usage");
assert_eq!((cpu.qos.as_str(), cpu.qos_source), ("sampled", "default"));
assert_eq!(cpu.rate_hz, 1.0);
assert_eq!(cpu.body_source, "placeholder");
assert!(
cpu.note.as_deref().unwrap_or("").contains("no schema"),
"{:?}",
cpu.note
);
let boom = &plan[2];
assert_eq!(boom.class, "events");
assert_eq!(boom.events_cap, Some(1), "rare = 1/h caps a 10s run at 1");
assert!(boom.unique_chunk.is_some(), "events keys are write-once");
assert!(
boom.note.as_deref().unwrap_or("").contains("{id}"),
"the synthesized var is stated: {:?}",
boom.note
);
}
#[tokio::test]
async fn events_keys_get_a_fresh_id_where_the_var_was() {
for base in ["", "acme", "acme/fleet-a"] {
let plan = plan_for(base).await;
let boom = plan.iter().find(|e| e.class == "events").unwrap();
let k1 = unique_key(boom, 42, 0);
let k2 = unique_key(boom, 42, 1);
assert_ne!(k1, k2, "each send gets its own key ({base:?})");
let tail1: Vec<&str> = k1.split('/').collect();
let tail2: Vec<&str> = k2.split('/').collect();
assert_eq!(tail1.len(), tail2.len());
let diffs: Vec<usize> = (0..tail1.len()).filter(|&i| tail1[i] != tail2[i]).collect();
assert_eq!(diffs.len(), 1, "only the id chunk moves ({base:?})");
assert!(
k1.ends_with(tail1[diffs[0]]),
"the id is the declared {{id}} position ({base:?}): {k1}"
);
}
}
#[test]
fn the_marker_round_trips_through_the_doctors_detector() {
let m = synthetic_marker("zenctl gen", "h-abababababab", None);
let v: serde_json::Value = serde_json::from_slice(&m).unwrap();
assert_eq!(v["synthetic"], true);
assert_eq!(v["tool"], "zenctl gen");
assert_eq!(v["origin"], "h-abababababab");
assert!(v.get("fault").is_none(), "no fault key unless injecting");
let f = synthetic_marker("zenctl gen", "h-abababababab", Some("truncate"));
let v: serde_json::Value = serde_json::from_slice(&f).unwrap();
assert_eq!(v["fault"], "truncate");
}
#[test]
fn fault_kinds_parse_and_an_unknown_is_refused() {
for f in Fault::ALL {
assert_eq!(Fault::parse(f.as_str()).unwrap(), f);
}
let err = Fault::parse("scramble").unwrap_err().to_string();
assert!(err.contains("is not a known fault kind"), "{err}");
assert!(err.contains("truncate"), "the vocabulary is named: {err}");
}
#[tokio::test]
async fn faults_expand_the_plan_one_variant_per_kind_with_a_stated_delta() {
let slices =
SliceSet::from_slices(vec![zenkey::parse_slice(SLICES).expect("fixture parses")]);
let store = SchemaStore::new("", Duration::from_millis(100));
let mut spec = spec();
spec.faults = Fault::ALL.to_vec();
let plan = build_plan(None, &store, &slices, "", None, &spec)
.await
.expect("plan builds");
assert_eq!(plan.len(), 3 * 7);
assert!(
plan.iter()
.all(|e| e.fault.is_some() && e.fault_delta.is_some()),
"every faulted entry names its kind and delta"
);
let health: Vec<&GenPlanEntry> = plan
.iter()
.filter(|e| e.key.starts_with("v1/h-abababababab/state/demo/health"))
.collect();
assert_eq!(health.len(), 7);
let unregistered = health
.iter()
.find(|e| e.fault == Some(Fault::UnregisteredKey))
.unwrap();
assert_eq!(
unregistered.key,
"v1/h-abababababab/state/demo/health/unregistered"
);
let wrong_qos = health
.iter()
.find(|e| e.fault == Some(Fault::WrongQos))
.unwrap();
assert_ne!(
wrong_qos.qos, "transition",
"the declared profile is not honoured"
);
let missing_enc = health
.iter()
.find(|e| e.fault == Some(Fault::MissingEncoding))
.unwrap();
assert!(
missing_enc.encoding.is_none(),
"the wire encoding is dropped"
);
let truncate = health
.iter()
.find(|e| e.fault == Some(Fault::Truncate))
.unwrap();
assert_eq!(truncate.qos, "transition");
assert!(truncate.key.ends_with("/health"));
}
#[test]
fn body_faults_perturb_the_encoded_bytes() {
let valid = br#"{"ok":true,"load":3}"#.to_vec();
let truncated = Fault::Truncate.perturb_body(valid.clone());
assert_eq!(truncated.len(), valid.len() / 2, "half the bytes survive");
let wrong = Fault::WrongType.perturb_body(valid.clone());
let v: serde_json::Value = serde_json::from_slice(&wrong).unwrap();
assert!(v.is_string(), "a bare string where an object was declared");
let extra = Fault::ExtraField.perturb_body(valid.clone());
let v: serde_json::Value = serde_json::from_slice(&extra).unwrap();
assert_eq!(v["_fault"], true, "the undeclared field rides");
assert_eq!(v["ok"], true, "the valid fields survive alongside it");
}
}