use crate::broker::worker::Worker;
use crate::sched::batch::{encode_result, BatchResult, ResultStatus};
pub(crate) const PLACEMENT_HEADER: &str = "X-Zakuro-Placement";
pub(crate) const MESH: &str = "mesh";
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Target {
Own,
Peer { url: String },
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct NodeGroup {
pub target: Target,
pub node: String,
pub name: String,
pub workers: usize,
}
pub(crate) fn group_by_node<'a, I: IntoIterator<Item = &'a Worker>>(
ranked: I,
own_fp: &str,
is_own: impl Fn(&Worker) -> bool,
url_of: impl Fn(&str) -> Option<String>,
) -> Vec<NodeGroup> {
let mut groups: Vec<NodeGroup> = Vec::new();
for w in ranked {
let (node, target) = if is_own(w) {
(format!("zc://node-{own_fp}"), Target::Own)
} else if w.node_fp.is_empty() || w.node_fp == own_fp {
continue;
} else {
match url_of(&w.node_fp) {
Some(url) => (format!("zc://node-{}", w.node_fp), Target::Peer { url }),
None => continue,
}
};
match groups.iter_mut().find(|g| g.node == node) {
Some(g) => g.workers += 1,
None => groups.push(NodeGroup {
name: w.source_node.clone().unwrap_or_else(|| node.clone()),
target,
node,
workers: 1,
}),
}
}
groups
}
pub(crate) fn allocate(weights: &[usize], n: usize) -> Vec<usize> {
let total: usize = weights.iter().sum();
if total == 0 {
return vec![0; weights.len()];
}
let mut counts: Vec<usize> = weights.iter().map(|w| n * w / total).collect();
let mut remainders: Vec<(usize, usize)> = weights
.iter()
.enumerate()
.map(|(i, w)| (i, (n * w) % total))
.collect();
remainders.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0)));
let left = n - counts.iter().sum::<usize>();
for (i, _) in remainders.iter().take(left) {
counts[*i] += 1;
}
counts
}
#[derive(Debug, Clone)]
pub(crate) struct ShareReply {
pub status: u16,
pub headers: Vec<(String, String)>,
pub body: Vec<u8>,
}
#[derive(Debug, Clone)]
pub(crate) struct ShareOutcome {
pub group: NodeGroup,
pub tasks: Vec<u32>,
pub reply: Result<ShareReply, String>,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub(crate) struct ChargeView {
pub node: String,
pub name: String,
pub batch_id: Option<String>,
pub jobs: usize,
pub cost: f64,
pub remaining: Option<f64>,
pub worker: Option<String>,
pub service_ms: Option<f64>,
}
#[derive(Debug, Clone)]
pub(crate) struct Merged {
pub body: Vec<u8>,
pub cost: f64,
pub charges: Vec<ChargeView>,
}
fn header<'a>(h: &'a [(String, String)], name: &str) -> Option<&'a str> {
h.iter()
.find(|(k, _)| k.eq_ignore_ascii_case(name))
.map(|(_, v)| v.as_str())
}
pub(crate) fn failed_frames(ids: &[u32], why: &str) -> Vec<u8> {
let mut out = Vec::new();
for id in ids {
out.extend(encode_result(&BatchResult {
id: *id,
status: ResultStatus::Failed,
body: why.as_bytes().to_vec(),
}));
}
out
}
pub(crate) fn merge(outcomes: Vec<ShareOutcome>) -> Merged {
let mut body = Vec::new();
let mut cost = 0.0;
let mut charges = Vec::new();
for o in outcomes {
match o.reply {
Ok(r) if r.status == 200 => {
let frames = match crate::sched::batch::decode_results(&r.body) {
Ok((frames, used)) if used == r.body.len() => frames,
_ => {
body.extend(failed_frames(
&o.tasks,
&format!("{}: the reply was not a batch result", o.group.name),
));
continue;
}
};
let mut answered = std::collections::HashSet::new();
for f in frames {
if o.tasks.contains(&f.id) && answered.insert(f.id) {
body.extend(encode_result(&f));
}
}
let missing: Vec<u32> = o
.tasks
.iter()
.copied()
.filter(|id| !answered.contains(id))
.collect();
body.extend(failed_frames(
&missing,
&format!("{}: no result returned for this task", o.group.name),
));
let c = header(&r.headers, "X-Zakuro-Cost")
.and_then(|v| v.parse::<f64>().ok())
.unwrap_or(0.0);
cost += c;
charges.push(ChargeView {
node: o.group.node.clone(),
name: header(&r.headers, "X-Zakuro-Node-Name")
.filter(|n| !n.is_empty())
.map(str::to_string)
.unwrap_or_else(|| o.group.name.clone()),
batch_id: header(&r.headers, "X-Zakuro-Batch-Id").map(str::to_string),
jobs: o.tasks.len(),
cost: c,
remaining: header(&r.headers, "X-Zakuro-Balance")
.and_then(|v| v.parse::<f64>().ok()),
worker: header(&r.headers, "X-Zakuro-Worker").map(str::to_string),
service_ms: header(&r.headers, "X-Zakuro-Service-Ms")
.and_then(|v| v.parse::<f64>().ok()),
});
}
Ok(r) => {
let detail: String = String::from_utf8_lossy(&r.body).chars().take(160).collect();
body.extend(failed_frames(
&o.tasks,
&format!("{}: HTTP {}: {}", o.group.name, r.status, detail.trim()),
));
}
Err(e) => body.extend(failed_frames(&o.tasks, &format!("{}: {e}", o.group.name))),
}
}
Merged {
body,
cost,
charges,
}
}
const SHARE_SLACK_SECS: f64 = 30.0;
pub(crate) fn share_timeout_secs(task_timeout_secs: f64, default_secs: f64) -> f64 {
let per_task = if task_timeout_secs > 0.0 {
task_timeout_secs + 5.0
} else {
default_secs
};
per_task + SHARE_SLACK_SECS
}
pub(crate) fn share_requirements(raw: Option<&str>, k: usize, n: usize) -> Option<String> {
let raw = raw?;
let mut v: serde_json::Value = match serde_json::from_str(raw) {
Ok(v) => v,
Err(_) => return Some(raw.to_string()),
};
match v.get("budget").and_then(|b| b.as_f64()) {
Some(budget) if n > 0 => {
v["budget"] = serde_json::json!(budget * k as f64 / n as f64);
Some(v.to_string())
}
_ => Some(raw.to_string()),
}
}
pub(crate) fn yen_header(v: f64) -> String {
format!("{:.6}", if v == 0.0 { 0.0 } else { v })
}
pub(crate) async fn await_share(
group: NodeGroup,
tasks: Vec<u32>,
handle: tokio::task::JoinHandle<Result<ShareReply, String>>,
) -> ShareOutcome {
let reply = match handle.await {
Ok(r) => r,
Err(e) => Err(format!("share task panicked: {e}")),
};
ShareOutcome {
group,
tasks,
reply,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::broker::worker::Worker;
fn w(name: &str, fp: &str, host: &str) -> Worker {
let mut w = Worker::new(
format!("id-{name}"),
name.into(),
format!("http://{host}:3960"),
"zakuro".into(),
);
w.node_fp = fp.into();
w
}
#[test]
fn workers_group_by_node_in_ranked_order() {
let ranked = [
w("b1", "bbbb", "10.0.0.2"),
w("own", "", "127.0.0.1"),
w("b2", "bbbb", "10.0.0.2"),
w("c1", "cccc", "10.0.0.3"),
];
let groups = group_by_node(
ranked.iter(),
"aaaa",
|w| w.uri.contains("127.0.0.1"),
|fp| Some(format!("http://peer-{fp}:9000")),
);
let summary: Vec<(String, usize)> =
groups.iter().map(|g| (g.node.clone(), g.workers)).collect();
assert_eq!(
summary,
vec![
("zc://node-bbbb".to_string(), 2),
("zc://node-aaaa".to_string(), 1),
("zc://node-cccc".to_string(), 1),
]
);
assert_eq!(groups[1].target, Target::Own);
assert_eq!(
groups[0].target,
Target::Peer {
url: "http://peer-bbbb:9000".into()
}
);
}
#[test]
fn unaddressable_peer_workers_are_left_out() {
let ranked = [w("anon", "", "10.0.0.9"), w("lost", "dddd", "10.0.0.4")];
let groups = group_by_node(ranked.iter(), "aaaa", |_| false, |_| None);
assert!(groups.is_empty(), "{groups:?}");
}
#[test]
fn tasks_split_in_proportion_and_always_add_up() {
assert_eq!(
allocate(&[1, 1], 3),
vec![2, 1],
"ties go to the better-ranked node"
);
assert_eq!(allocate(&[3, 1], 4), vec![3, 1]);
assert_eq!(allocate(&[2, 1, 1], 1), vec![1, 0, 0]);
assert_eq!(allocate(&[5, 3, 2], 7), vec![4, 2, 1]);
for n in 0..50 {
assert_eq!(allocate(&[4, 2, 1], n).iter().sum::<usize>(), n, "n={n}");
}
assert_eq!(allocate(&[], 5), Vec::<usize>::new());
assert_eq!(allocate(&[0, 0], 5), vec![0, 0]);
}
use crate::sched::batch::{decode_results, encode_result, BatchResult, ResultStatus};
fn group(name: &str, target: Target) -> NodeGroup {
NodeGroup {
target,
node: format!("zc://node-{name}"),
name: name.into(),
workers: 1,
}
}
fn ok_reply(ids: &[u32], cost: &str, worker: &str) -> ShareReply {
let mut body = Vec::new();
for id in ids {
body.extend(encode_result(&BatchResult {
id: *id,
status: ResultStatus::Ok,
body: b"r".to_vec(),
}));
}
ShareReply {
status: 200,
headers: vec![
("x-zakuro-cost".into(), cost.into()),
("x-zakuro-batch-id".into(), format!("b-{worker}")),
("x-zakuro-balance".into(), "9.5".into()),
("x-zakuro-worker".into(), worker.into()),
("x-zakuro-service-ms".into(), "12.5".into()),
],
body,
}
}
#[test]
fn replies_merge_into_one_body_with_a_charge_per_node() {
let merged = merge(vec![
ShareOutcome {
group: group("aaaa", Target::Own),
tasks: vec![1, 2],
reply: Ok(ok_reply(&[1, 2], "0.002000", "own-w")),
},
ShareOutcome {
group: group("bbbb", Target::Peer { url: "u".into() }),
tasks: vec![3],
reply: Ok(ok_reply(&[3], "0.001000", "peer-w")),
},
]);
let (results, _) = decode_results(&merged.body).unwrap();
assert_eq!(results.len(), 3);
assert!(results.iter().all(|r| r.status == ResultStatus::Ok));
assert!((merged.cost - 0.003).abs() < 1e-9);
assert_eq!(merged.charges.len(), 2);
assert_eq!(merged.charges[1].worker.as_deref(), Some("peer-w"));
assert_eq!(merged.charges[1].jobs, 1);
assert_eq!(merged.charges[0].remaining, Some(9.5));
assert_eq!(merged.charges[0].batch_id.as_deref(), Some("b-own-w"));
}
#[test]
fn a_failed_share_fails_only_its_own_tasks() {
let merged = merge(vec![
ShareOutcome {
group: group("aaaa", Target::Own),
tasks: vec![1],
reply: Ok(ok_reply(&[1], "0.001000", "own-w")),
},
ShareOutcome {
group: group("bbbb", Target::Peer { url: "u".into() }),
tasks: vec![2, 3],
reply: Err("connection refused".into()),
},
ShareOutcome {
group: group("cccc", Target::Peer { url: "u".into() }),
tasks: vec![4],
reply: Ok(ShareReply {
status: 402,
headers: vec![],
body: b"Insufficient balance".to_vec(),
}),
},
]);
let (results, _) = decode_results(&merged.body).unwrap();
let failed: Vec<(u32, String)> = results
.iter()
.filter(|r| r.status == ResultStatus::Failed)
.map(|r| (r.id, String::from_utf8_lossy(&r.body).to_string()))
.collect();
assert_eq!(failed.len(), 3, "{failed:?}");
assert!(failed
.iter()
.any(|(id, why)| *id == 2 && why.contains("bbbb: connection refused")));
assert!(failed
.iter()
.any(|(id, why)| *id == 4 && why.contains("cccc: HTTP 402")));
assert_eq!(
merged.charges.len(),
1,
"only the share that ran is charged"
);
assert!((merged.cost - 0.001).abs() < 1e-9);
}
#[test]
fn failed_frames_answer_every_id() {
let (results, _) = decode_results(&failed_frames(&[7, 8], "no eligible worker")).unwrap();
assert_eq!(results.iter().map(|r| r.id).collect::<Vec<_>>(), vec![7, 8]);
assert!(results.iter().all(|r| r.status == ResultStatus::Failed));
}
#[test]
fn a_synced_copy_of_our_own_nodes_worker_is_left_out() {
let mut synced = w("other-broker-w", "aaaa", "127.0.0.1");
synced.source_node = Some("node-aaaa".into());
let groups = group_by_node(
[synced].iter(),
"aaaa",
|w| w.source_node.is_none(),
|fp| Some(format!("http://peer-{fp}:9000")),
);
assert!(groups.is_empty(), "{groups:?}");
}
#[test]
fn a_share_waits_longer_than_its_tasks_may_run() {
assert!(share_timeout_secs(300.0, 60.0) > 305.0);
assert!(
share_timeout_secs(0.0, 60.0) > 60.0,
"default task timeout, plus slack"
);
}
#[test]
fn a_budget_is_split_across_shares_in_proportion() {
let raw = r#"{"cpus":1.0,"budget":1.0,"timeout_secs":300.0}"#;
let a: serde_json::Value =
serde_json::from_str(&share_requirements(Some(raw), 3, 4).unwrap()).unwrap();
let b: serde_json::Value =
serde_json::from_str(&share_requirements(Some(raw), 1, 4).unwrap()).unwrap();
assert!((a["budget"].as_f64().unwrap() - 0.75).abs() < 1e-12);
assert!((b["budget"].as_f64().unwrap() - 0.25).abs() < 1e-12);
assert_eq!(a["timeout_secs"], 300.0, "the rest is passed through");
assert_eq!(share_requirements(None, 1, 4), None);
let plain = r#"{"cpus":1.0}"#;
assert_eq!(
share_requirements(Some(plain), 1, 4).as_deref(),
Some(plain)
);
}
#[test]
fn a_share_body_is_checked_before_it_is_merged() {
let garbage = ShareReply {
status: 200,
headers: vec![("x-zakuro-cost".into(), "0.001".into())],
body: b"<html>502 Bad Gateway</html>".to_vec(),
};
let partial = ok_reply(&[5, 99], "0.002000", "w");
let merged = merge(vec![
ShareOutcome {
group: group("bbbb", Target::Peer { url: "u".into() }),
tasks: vec![1, 2],
reply: Ok(garbage),
},
ShareOutcome {
group: group("cccc", Target::Peer { url: "u".into() }),
tasks: vec![5, 6],
reply: Ok(partial),
},
]);
let (results, used) = decode_results(&merged.body).unwrap();
assert_eq!(
used,
merged.body.len(),
"the merged body is all well-formed frames"
);
let mut ids: Vec<u32> = results.iter().map(|r| r.id).collect();
ids.sort_unstable();
assert_eq!(ids, vec![1, 2, 5, 6], "99 was never sent to cccc");
let status = |id: u32| results.iter().find(|r| r.id == id).unwrap().status;
assert_eq!(status(1), ResultStatus::Failed);
assert_eq!(status(5), ResultStatus::Ok);
assert_eq!(status(6), ResultStatus::Failed, "unanswered");
assert_eq!(
merged.charges.len(),
1,
"a garbage reply settled nothing we can show"
);
}
#[test]
fn a_charge_uses_the_node_name_its_broker_reports() {
let mut r = ok_reply(&[1], "0.001000", "w");
r.headers
.push(("x-zakuro-node-name".into(), "zc-worker-i9-1".into()));
let merged = merge(vec![ShareOutcome {
group: group("e18d", Target::Peer { url: "u".into() }),
tasks: vec![1],
reply: Ok(r),
}]);
assert_eq!(merged.charges[0].name, "zc-worker-i9-1");
}
#[test]
fn a_zero_charge_is_written_without_a_sign() {
assert_eq!(yen_header(-0.0), "0.000000");
assert_eq!(yen_header(0.0015), "0.001500");
}
#[tokio::test]
async fn a_panicked_share_still_answers_its_tasks() {
let handle = tokio::spawn(async { panic!("boom") });
let outcome = await_share(
group("bbbb", Target::Peer { url: "u".into() }),
vec![3, 4],
handle,
)
.await;
assert_eq!(outcome.tasks, vec![3, 4]);
assert!(outcome.reply.unwrap_err().contains("panicked"));
}
}