use std::net::{SocketAddr, ToSocketAddrs};
use std::sync::{Arc, Mutex, OnceLock};
use dashmap::DashMap;
use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName, UnixTime};
use tokio::runtime::Runtime;
use std::collections::HashSet;
use std::sync::mpsc;
use std::thread;
use std::time::Duration;
use super::node_identity::NodeKey;
use super::server::{quic_offer_gate, run_peer_offer, PeerGateFlags, PeerOfferOutcome};
use super::task_board::{SubscribeMessage, TaskOffer, TaskReject, TaskResult};
use super::BrokerState;
type ExecuteFn =
Arc<dyn Fn(&BrokerState, &TaskOffer, bool) -> Result<TaskResult, TaskReject> + Send + Sync>;
type OfferReceiver = mpsc::Receiver<(
TaskOffer,
mpsc::Sender<(String, Result<TaskResult, TaskReject>)>,
)>;
fn parse_persisted_key(key_der: Vec<u8>) -> Option<PrivateKeyDer<'static>> {
PrivateKeyDer::try_from(key_der).ok()
}
fn quic_seed(peer_key: &str) -> [u8; 32] {
let mut input = Vec::with_capacity(12 + peer_key.len());
input.extend_from_slice(b"zc-quic-pin\0");
input.extend_from_slice(peer_key.as_bytes());
let digest = ring::digest::digest(&ring::digest::SHA256, &input);
let mut out = [0u8; 32];
out.copy_from_slice(digest.as_ref());
out
}
fn ed25519_pkcs8(seed: &[u8; 32]) -> Vec<u8> {
const PREFIX: [u8; 16] = [
0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04,
0x20,
];
let mut der = Vec::with_capacity(48);
der.extend_from_slice(&PREFIX);
der.extend_from_slice(seed);
der
}
fn derive_quic_identity(
peer_key: &str,
) -> Result<(CertificateDer<'static>, PrivateKeyDer<'static>), String> {
let seed = quic_seed(peer_key);
let pkcs8 = ed25519_pkcs8(&seed);
let pkcs8_der = PrivatePkcs8KeyDer::from(pkcs8.clone());
let key_pair = rcgen::KeyPair::from_pkcs8_der_and_sign_algo(&pkcs8_der, &rcgen::PKCS_ED25519)
.map_err(|e| format!("derive keypair: {}", e))?;
let mut params = rcgen::CertificateParams::new(vec!["zakuro-broker".to_string()])
.map_err(|e| format!("cert params: {}", e))?;
params.serial_number = Some(rcgen::SerialNumber::from_slice(&[0x01]));
params.not_before = rcgen::date_time_ymd(2020, 1, 1);
params.not_after = rcgen::date_time_ymd(2100, 1, 1);
params.distinguished_name = {
let mut dn = rcgen::DistinguishedName::new();
dn.push(rcgen::DnType::CommonName, "zakuro-broker");
dn
};
let cert = params
.self_signed(&key_pair)
.map_err(|e| format!("self-sign: {}", e))?;
let cert_der = CertificateDer::from(cert.der().to_vec());
let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(pkcs8));
Ok((cert_der, key))
}
fn cert_sha256(cert: &CertificateDer) -> [u8; 32] {
let digest = ring::digest::digest(&ring::digest::SHA256, cert.as_ref());
let mut out = [0u8; 32];
out.copy_from_slice(digest.as_ref());
out
}
fn generate_self_signed() -> (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>) {
if let Some(home) = std::env::var("HOME")
.ok()
.or_else(|| std::env::var("USERPROFILE").ok())
{
let dir = std::path::Path::new(&home).join(".zakuro");
let cert_path = dir.join("quic_cert.der");
let key_path = dir.join("quic_key.der");
if cert_path.exists() && key_path.exists() {
if let (Ok(cert_der), Ok(key_der)) =
(std::fs::read(&cert_path), std::fs::read(&key_path))
{
if let Some(key) = parse_persisted_key(key_der) {
return (vec![CertificateDer::from(cert_der)], key);
}
eprintln!("[quic] persisted QUIC key is invalid; regenerating self-signed cert");
}
}
let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_string()])
.expect("self-signed cert generation failed");
let cert_der: Vec<u8> = cert.cert.der().to_vec();
let key_der: Vec<u8> = cert.key_pair.serialize_der();
let _ = std::fs::create_dir_all(&dir);
let _ = std::fs::write(&cert_path, &cert_der);
let _ = std::fs::write(&key_path, &key_der);
if let Some(key) = parse_persisted_key(key_der) {
return (vec![CertificateDer::from(cert_der)], key);
}
}
let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_string()])
.expect("self-signed cert generation failed");
let cert_der: Vec<u8> = cert.cert.der().to_vec();
let key_der: Vec<u8> = cert.key_pair.serialize_der();
(
vec![CertificateDer::from(cert_der)],
PrivateKeyDer::try_from(key_der).expect("rcgen-generated QUIC key was not parseable"),
)
}
fn make_server_config(
certs: Vec<CertificateDer<'static>>,
key: PrivateKeyDer<'static>,
) -> quinn::ServerConfig {
let mut sc = rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(certs, key)
.expect("bad server TLS config");
sc.alpn_protocols = vec![b"zk-quic".to_vec()];
let quic_sc = quinn::crypto::rustls::QuicServerConfig::try_from(sc)
.expect("QUIC requires a rustls config supporting TLS 1.3");
let mut cfg = quinn::ServerConfig::with_crypto(Arc::new(quic_sc));
cfg.transport_config(Arc::new(task_feed_transport_config()));
cfg
}
fn task_feed_transport_config() -> quinn::TransportConfig {
let mut tc = quinn::TransportConfig::default();
tc.keep_alive_interval(Some(Duration::from_secs(5)));
tc.max_idle_timeout(Some(
Duration::from_secs(30)
.try_into()
.expect("30s is a valid QUIC idle timeout"),
));
tc
}
fn resolve_peer_addr(host: &str, port: u16) -> Option<SocketAddr> {
let candidates: Vec<SocketAddr> = (host, port).to_socket_addrs().ok()?.collect();
candidates
.iter()
.find(|a| a.is_ipv4())
.or_else(|| candidates.first())
.copied()
}
fn make_client_config(
verifier: Arc<dyn rustls::client::danger::ServerCertVerifier>,
) -> quinn::ClientConfig {
let mut cc = rustls::ClientConfig::builder()
.dangerous()
.with_custom_certificate_verifier(verifier)
.with_no_client_auth();
cc.alpn_protocols = vec![b"zk-quic".to_vec()];
let quic_cc = quinn::crypto::rustls::QuicClientConfig::try_from(cc)
.expect("QUIC requires a rustls config supporting TLS 1.3");
let mut cfg = quinn::ClientConfig::new(Arc::new(quic_cc));
cfg.transport_config(Arc::new(task_feed_transport_config()));
cfg
}
fn build_tls_configs(peer_key: &str) -> (quinn::ServerConfig, quinn::ClientConfig) {
if !peer_key.is_empty() {
match derive_quic_identity(peer_key) {
Ok((cert, key)) => {
let pin = cert_sha256(&cert);
let server_cfg = make_server_config(vec![cert], key);
let client_cfg = make_client_config(Arc::new(PinnedVerify { expected: pin }));
return (server_cfg, client_cfg);
}
Err(e) => {
eprintln!(
"[quic] peer-key cert derivation failed ({e}); falling back to unpinned TLS"
);
}
}
}
let (certs, key) = generate_self_signed();
(
make_server_config(certs, key),
make_client_config(Arc::new(SkipVerify)),
)
}
#[derive(Debug)]
struct SkipVerify;
impl rustls::client::danger::ServerCertVerifier for SkipVerify {
fn verify_server_cert(
&self,
_end_entity: &CertificateDer<'_>,
_intermediates: &[CertificateDer<'_>],
_server_name: &ServerName<'_>,
_ocsp_response: &[u8],
_now: UnixTime,
) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
Ok(rustls::client::danger::ServerCertVerified::assertion())
}
fn verify_tls12_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
}
fn verify_tls13_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
}
fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
rustls::crypto::ring::default_provider()
.signature_verification_algorithms
.supported_schemes()
}
}
#[derive(Debug)]
struct PinnedVerify {
expected: [u8; 32],
}
impl rustls::client::danger::ServerCertVerifier for PinnedVerify {
fn verify_server_cert(
&self,
end_entity: &CertificateDer<'_>,
_intermediates: &[CertificateDer<'_>],
_server_name: &ServerName<'_>,
_ocsp_response: &[u8],
_now: UnixTime,
) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
if cert_sha256(end_entity) == self.expected {
Ok(rustls::client::danger::ServerCertVerified::assertion())
} else {
Err(rustls::Error::InvalidCertificate(
rustls::CertificateError::ApplicationVerificationFailure,
))
}
}
fn verify_tls12_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
}
fn verify_tls13_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
}
fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
rustls::crypto::ring::default_provider()
.signature_verification_algorithms
.supported_schemes()
}
}
async fn write_msg(send: &mut quinn::SendStream, payload: &[u8]) -> Result<(), String> {
let len = (payload.len() as u32).to_be_bytes();
send.write_all(&len).await.map_err(|e| e.to_string())?;
send.write_all(payload).await.map_err(|e| e.to_string())?;
send.finish().map_err(|e| e.to_string())?;
Ok(())
}
async fn write_msg_keep_open(send: &mut quinn::SendStream, payload: &[u8]) -> Result<(), String> {
let len = (payload.len() as u32).to_be_bytes();
send.write_all(&len).await.map_err(|e| e.to_string())?;
send.write_all(payload).await.map_err(|e| e.to_string())?;
Ok(())
}
const MAX_QUIC_MSG_BYTES: usize = 8 * 1024 * 1024;
fn cap_msg_len(len_bytes: [u8; 4]) -> Result<usize, String> {
let len = u32::from_be_bytes(len_bytes) as usize;
if len > MAX_QUIC_MSG_BYTES {
return Err(format!(
"message too large: {} bytes (max {})",
len, MAX_QUIC_MSG_BYTES
));
}
Ok(len)
}
async fn read_msg(recv: &mut quinn::RecvStream) -> Result<Vec<u8>, String> {
let mut len_buf = [0u8; 4];
recv.read_exact(&mut len_buf)
.await
.map_err(|e| e.to_string())?;
let len = cap_msg_len(len_buf)?;
const CHUNK: usize = 64 * 1024;
let mut buf: Vec<u8> = Vec::with_capacity(len.min(CHUNK));
let mut remaining = len;
let mut chunk = [0u8; CHUNK];
while remaining > 0 {
let want = remaining.min(CHUNK);
recv.read_exact(&mut chunk[..want])
.await
.map_err(|e| e.to_string())?;
buf.extend_from_slice(&chunk[..want]);
remaining -= want;
}
Ok(buf)
}
fn peer_key_ok(expected: &str, provided: &str) -> bool {
expected.is_empty() || expected == provided
}
type SubscriberOfferTx = mpsc::Sender<(
TaskOffer,
mpsc::Sender<(String, Result<TaskResult, TaskReject>)>,
)>;
pub struct QuicTransport {
runtime: Runtime,
endpoint: quinn::Endpoint,
connections: DashMap<SocketAddr, quinn::Connection>,
subscriber_txs: DashMap<String, SubscriberOfferTx>,
peer_key: String,
node_key: Arc<NodeKey>,
gate_flags_override: OnceLock<PeerGateFlags>,
}
impl QuicTransport {
pub fn new(bind_addr: SocketAddr, state: Arc<BrokerState>) -> Result<Arc<Self>, String> {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.map_err(|e| format!("tokio runtime: {}", e))?;
let peer_key = state.peer_manager.peer_key().to_string();
let (server_cfg, client_cfg) = build_tls_configs(&peer_key);
let endpoint = runtime.block_on(async {
let mut ep = quinn::Endpoint::server(server_cfg, bind_addr)
.map_err(|e| format!("QUIC bind {}: {}", bind_addr, e))?;
ep.set_default_client_config(client_cfg);
Ok::<_, String>(ep)
})?;
let transport = Arc::new(Self {
runtime,
endpoint,
connections: DashMap::new(),
subscriber_txs: DashMap::new(),
peer_key,
node_key: state.node_key.clone(),
gate_flags_override: OnceLock::new(),
});
let t = transport.clone();
transport.runtime.spawn(async move {
accept_loop(t, state).await;
});
Ok(transport)
}
fn gate_flags(&self) -> PeerGateFlags {
self.gate_flags_override
.get()
.copied()
.unwrap_or_else(PeerGateFlags::from_env)
}
#[cfg(test)]
fn set_gate_flags(&self, flags: PeerGateFlags) {
let _ = self.gate_flags_override.set(flags);
}
pub fn offer_task(
&self,
peer_addr: SocketAddr,
offer: &TaskOffer,
) -> Result<TaskResult, String> {
self.runtime
.block_on(self.offer_task_inner(peer_addr, offer))
}
async fn offer_task_inner(
&self,
peer_addr: SocketAddr,
offer: &TaskOffer,
) -> Result<TaskResult, String> {
let conn = self.get_or_connect(peer_addr).await?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| format!("open stream: {}", e))?;
let mut offer = offer.clone();
offer.peer_key = self.peer_key.clone();
offer.sign_for_quic(&self.node_key);
let payload = serde_json::to_vec(&offer).map_err(|e| e.to_string())?;
write_msg(&mut send, &payload).await?;
let resp = read_msg(&mut recv).await?;
if let Ok(result) = serde_json::from_slice::<TaskResult>(&resp) {
return Ok(result);
}
if let Ok(reject) = serde_json::from_slice::<TaskReject>(&resp) {
return Err(format!("peer rejected: {}", reject.reason));
}
Err("invalid peer response".into())
}
async fn get_or_connect(&self, addr: SocketAddr) -> Result<quinn::Connection, String> {
if let Some(c) = self.connections.get(&addr) {
if c.close_reason().is_none() {
return Ok(c.clone());
}
}
self.connections.remove(&addr);
let conn = self
.endpoint
.connect(addr, "localhost")
.map_err(|e| format!("connect {}: {}", addr, e))?
.await
.map_err(|e| format!("handshake {}: {}", addr, e))?;
self.connections.insert(addr, conn.clone());
Ok(conn)
}
pub fn local_addr(&self) -> Option<SocketAddr> {
self.endpoint.local_addr().ok()
}
pub fn broadcast_offer_to_subscribers(
&self,
offer: &TaskOffer,
) -> Option<(String, TaskResult)> {
let subs: Vec<(String, SubscriberOfferTx)> = self
.subscriber_txs
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
if subs.is_empty() {
return None;
}
let (reply_tx, reply_rx) = mpsc::channel();
let mut dead: Vec<String> = Vec::new();
for (url, offer_tx) in &subs {
if offer_tx.send((offer.clone(), reply_tx.clone())).is_err() {
dead.push(url.clone());
}
}
for url in &dead {
self.subscriber_txs.remove(url);
eprintln!(" [QUIC] pruned disconnected subscriber {}", url);
}
drop(reply_tx); let n = subs.len() - dead.len();
for _ in 0..n {
if let Ok((url, Ok(result))) = reply_rx.recv() {
return Some((url, result));
}
}
None
}
pub fn subscriber_urls(&self) -> Vec<String> {
self.subscriber_txs
.iter()
.map(|e| e.key().clone())
.collect()
}
pub fn connect_and_subscribe(
&self,
peer_addr: SocketAddr,
our_url: &str,
) -> Result<(quinn::SendStream, quinn::RecvStream), String> {
self.runtime
.block_on(self.connect_and_subscribe_async(peer_addr, our_url))
}
async fn connect_and_subscribe_async(
&self,
peer_addr: SocketAddr,
our_url: &str,
) -> Result<(quinn::SendStream, quinn::RecvStream), String> {
let conn = self
.endpoint
.connect(peer_addr, "localhost")
.map_err(|e| format!("connect: {}", e))?
.await
.map_err(|e| format!("handshake: {}", e))?;
let (mut send, recv) = conn
.open_bi()
.await
.map_err(|e| format!("open stream: {}", e))?;
let msg = serde_json::to_vec(&SubscribeMessage {
action: "subscribe".to_string(),
peer_url: our_url.to_string(),
peer_key: self.peer_key.clone(),
})
.map_err(|e| e.to_string())?;
write_msg_keep_open(&mut send, &msg).await?;
Ok((send, recv))
}
}
pub fn run_subscription_client(state: Arc<BrokerState>, our_url: String, execute_fn: ExecuteFn) {
let spawned = Arc::new(Mutex::new(HashSet::<String>::new()));
thread::spawn(move || {
let mut interval = 0u32;
loop {
thread::sleep(Duration::from_millis(if interval < 20 {
2000
} else {
10000
}));
interval = interval.saturating_add(1);
let transport = match state.quic.get().cloned() {
Some(t) => t,
None => continue,
};
let peer_urls: Vec<String> = state.peer_manager.peer_urls();
for peer_url in peer_urls {
if peer_url == our_url {
continue;
}
let quic_port = state.peer_manager.get_quic_port(&peer_url);
if quic_port == 0 {
continue;
}
let url_trimmed = peer_url
.trim_start_matches("http://")
.trim_start_matches("https://");
let host = url_trimmed.split(':').next().unwrap_or("127.0.0.1");
let connect_host = if host.starts_with("127.") {
"127.0.0.1"
} else {
host
};
let addr: SocketAddr = match resolve_peer_addr(connect_host, quic_port) {
Some(a) => a,
None => {
eprintln!(
" [QUIC subscribe] {}:{}: could not resolve — skipping",
connect_host, quic_port
);
continue;
}
};
{
let mut g = spawned.lock().unwrap();
if g.contains(&peer_url) {
continue;
}
g.insert(peer_url.clone());
}
let transport_c = transport.clone();
let state_c = state.clone();
let our_url_c = our_url.clone();
let execute_fn_c = execute_fn.clone();
let peer_url_c = peer_url.clone();
let spawned_c = spawned.clone();
thread::spawn(move || {
run_subscription_to_peer(
transport_c,
state_c,
addr,
&peer_url_c,
&our_url_c,
execute_fn_c,
);
spawned_c.lock().unwrap().remove(&peer_url_c);
});
}
}
});
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FeedAttempt {
ConnectFailed,
Closed,
}
fn keep_subscribing(cached_quic_port: u16) -> bool {
cached_quic_port > 0
}
fn run_feed_subscription(
cached_quic_port: impl Fn() -> u16,
mut attempt: impl FnMut() -> FeedAttempt,
pause: impl Fn(Duration),
) {
while keep_subscribing(cached_quic_port()) {
pause(match attempt() {
FeedAttempt::ConnectFailed => Duration::from_secs(5),
FeedAttempt::Closed => Duration::from_secs(2),
});
}
}
fn run_subscription_to_peer(
transport: Arc<QuicTransport>,
state: Arc<BrokerState>,
peer_addr: SocketAddr,
peer_url: &str,
our_url: &str,
execute_fn: ExecuteFn,
) {
let runtime = transport.runtime.handle().clone();
run_feed_subscription(
|| state.peer_manager.get_quic_port(peer_url),
|| {
let (mut send, mut recv) = match transport.connect_and_subscribe(peer_addr, our_url) {
Ok(pair) => pair,
Err(e) => {
eprintln!(" [QUIC subscribe] {}: {}", peer_addr, e);
return FeedAttempt::ConnectFailed;
}
};
eprintln!(" [QUIC subscribe] Subscribed to {} (task feed)", peer_addr);
while let Ok(msg) = runtime.block_on(read_msg(&mut recv)) {
let offer: TaskOffer = match serde_json::from_slice(&msg) {
Ok(o) => o,
Err(_) => continue,
};
let flags = transport.gate_flags();
let gated = gate_offer(&state, &msg, &offer, flags);
let resp = match gated
.map_or_else(|| execute_fn(&state, &offer, flags.require_voucher), Err)
{
Ok(r) => serde_json::to_vec(&r).unwrap_or_default(),
Err(rej) => serde_json::to_vec(&rej).unwrap_or_default(),
};
if runtime
.block_on(write_msg_keep_open(&mut send, &resp))
.is_err()
{
break;
}
}
FeedAttempt::Closed
},
thread::sleep,
);
eprintln!(
" [QUIC subscribe] {}: stopped; {} no longer advertises a QUIC port",
peer_addr, peer_url
);
}
impl Drop for QuicTransport {
fn drop(&mut self) {
self.endpoint.close(0u32.into(), b"shutdown");
}
}
async fn accept_loop(transport: Arc<QuicTransport>, state: Arc<BrokerState>) {
loop {
let incoming = match transport.endpoint.accept().await {
Some(i) => i,
None => break,
};
let transport_clone = transport.clone();
let state_clone = state.clone();
tokio::spawn(async move {
let conn = match incoming.await {
Ok(c) => c,
Err(e) => {
eprintln!(" [QUIC] accept error: {}", e);
return;
}
};
loop {
let stream = match conn.accept_bi().await {
Ok(s) => s,
Err(quinn::ConnectionError::ApplicationClosed(_)) => break,
Err(_) => break,
};
let t = transport_clone.clone();
let st = state_clone.clone();
tokio::spawn(async move {
handle_incoming_stream(stream, t, st).await;
});
}
});
}
}
async fn handle_incoming_stream(
(mut send, mut recv): (quinn::SendStream, quinn::RecvStream),
transport: Arc<QuicTransport>,
state: Arc<BrokerState>,
) {
let msg = match read_msg(&mut recv).await {
Ok(m) => m,
Err(_) => return,
};
let expected_key = state.peer_manager.peer_key();
if let Ok(sub) = serde_json::from_slice::<SubscribeMessage>(&msg) {
if sub.action == "subscribe" && !sub.peer_url.is_empty() {
if !peer_key_ok(expected_key, &sub.peer_key) {
eprintln!(
" [QUIC] rejected subscribe from {}: bad peer_key",
sub.peer_url
);
return;
}
let peer_url = sub.peer_url.clone();
let (offer_tx, offer_rx) = mpsc::channel();
let runtime_handle = transport.runtime.handle().clone();
let node_key = transport.node_key.clone();
thread::spawn(move || {
subscriber_loop(offer_rx, send, recv, peer_url, runtime_handle, node_key);
});
transport.subscriber_txs.insert(sub.peer_url, offer_tx);
return;
}
}
let offer: TaskOffer = match serde_json::from_slice(&msg) {
Ok(o) => o,
Err(e) => {
let r = TaskReject {
task_id: String::new(),
reason: format!("bad payload: {}", e),
};
let _ = write_msg(&mut send, &serde_json::to_vec(&r).unwrap()).await;
return;
}
};
if !peer_key_ok(expected_key, &offer.peer_key) {
let r = TaskReject {
task_id: offer.task_id.clone(),
reason: "unauthorized peer".into(),
};
let _ = write_msg(&mut send, &serde_json::to_vec(&r).unwrap()).await;
return;
}
let flags = transport.gate_flags();
if let Some(r) = gate_offer(&state, &msg, &offer, flags) {
let _ = write_msg(&mut send, &serde_json::to_vec(&r).unwrap()).await;
return;
}
handle_stream(send, recv, offer, state, flags.require_voucher).await;
}
fn gate_offer(
state: &Arc<BrokerState>,
raw: &[u8],
offer: &TaskOffer,
flags: PeerGateFlags,
) -> Option<TaskReject> {
quic_offer_gate(state, raw, offer, flags)
.err()
.map(|reason| TaskReject {
task_id: offer.task_id.clone(),
reason,
})
}
fn subscriber_loop(
offer_rx: OfferReceiver,
mut send: quinn::SendStream,
mut recv: quinn::RecvStream,
peer_url: String,
runtime_handle: tokio::runtime::Handle,
node_key: Arc<NodeKey>,
) {
while let Ok((mut offer, reply_tx)) = offer_rx.recv() {
offer.sign_for_quic(&node_key);
let payload = match serde_json::to_vec(&offer) {
Ok(p) => p,
Err(_) => continue,
};
let write_read = async {
write_msg_keep_open(&mut send, &payload).await?;
read_msg(&mut recv).await
};
let resp_bytes = match runtime_handle.block_on(write_read) {
Ok(b) => b,
Err(_) => break,
};
let result = if let Ok(r) = serde_json::from_slice::<TaskResult>(&resp_bytes) {
Ok(r)
} else if let Ok(rej) = serde_json::from_slice::<TaskReject>(&resp_bytes) {
Err(rej)
} else {
continue;
};
let _ = reply_tx.send((peer_url.clone(), result));
}
}
async fn handle_stream(
mut send: quinn::SendStream,
_recv: quinn::RecvStream,
offer: TaskOffer,
state: Arc<BrokerState>,
require_voucher: bool,
) {
let task_id = offer.task_id.clone();
let outcome =
tokio::task::spawn_blocking(move || run_peer_offer(&state, &offer, require_voucher)).await;
let resp_bytes = match outcome {
Ok(Ok(PeerOfferOutcome::Reserved(accept))) => serde_json::to_vec(&accept),
Ok(Ok(PeerOfferOutcome::Executed(result))) => serde_json::to_vec(&result),
Ok(Err(reject)) => serde_json::to_vec(&reject),
Err(e) => serde_json::to_vec(&TaskReject {
task_id,
reason: format!("internal: {}", e),
}),
};
if let Ok(bytes) = resp_bytes {
let _ = write_msg(&mut send, &bytes).await;
}
}
#[cfg(test)]
mod panic_fix_tests {
use super::{cap_msg_len, parse_persisted_key, peer_key_ok};
use crate::broker::task_board::{SubscribeMessage, TaskOffer};
#[test]
fn parse_persisted_key_rejects_garbage_without_panicking() {
assert!(parse_persisted_key(vec![0u8, 1, 2, 3]).is_none());
assert!(parse_persisted_key(Vec::new()).is_none());
}
#[test]
fn cap_msg_len_bounds() {
assert_eq!(cap_msg_len(1024u32.to_be_bytes()).unwrap(), 1024usize);
assert_eq!(cap_msg_len(0u32.to_be_bytes()).unwrap(), 0usize);
let over = (super::MAX_QUIC_MSG_BYTES as u32) + 1;
assert!(cap_msg_len(over.to_be_bytes()).is_err());
}
#[test]
fn empty_expected_allows_any_provided() {
assert!(peer_key_ok("", ""));
assert!(peer_key_ok("", "whatever"));
}
#[test]
fn matching_key_is_accepted() {
assert!(peer_key_ok("s3cret", "s3cret"));
}
#[test]
fn mismatched_or_missing_key_is_rejected() {
assert!(!peer_key_ok("s3cret", "wrong"));
assert!(!peer_key_ok("s3cret", ""));
}
#[test]
fn task_offer_peer_key_defaults_when_absent() {
let json = r#"{
"task_id":"t1","payload_b64":"","max_price_per_hour":1.0,
"estimated_duration_secs":1.0,"timeout_secs":10.0,
"requester_user_id":"u","source_broker":"n"
}"#;
let offer: TaskOffer = serde_json::from_str(json).unwrap();
assert_eq!(offer.peer_key, "");
}
#[test]
fn task_offer_peer_key_round_trips() {
let json = r#"{
"task_id":"t1","payload_b64":"","max_price_per_hour":1.0,
"estimated_duration_secs":1.0,"timeout_secs":10.0,
"requester_user_id":"u","source_broker":"n","peer_key":"abc"
}"#;
let offer: TaskOffer = serde_json::from_str(json).unwrap();
assert_eq!(offer.peer_key, "abc");
let back = serde_json::to_string(&offer).unwrap();
assert!(back.contains("\"peer_key\":\"abc\""));
}
#[test]
fn subscribe_message_peer_key_defaults_and_round_trips() {
let absent: SubscribeMessage =
serde_json::from_str(r#"{"action":"subscribe","peer_url":"http://x"}"#).unwrap();
assert_eq!(absent.peer_key, "");
let present: SubscribeMessage =
serde_json::from_str(r#"{"action":"subscribe","peer_url":"http://x","peer_key":"k"}"#)
.unwrap();
assert_eq!(present.peer_key, "k");
}
#[test]
fn dead_subscribers_are_pruned_on_send_failure() {
use dashmap::DashMap;
use std::sync::mpsc;
let subs: DashMap<String, mpsc::Sender<u8>> = DashMap::new();
let (live_tx, _live_rx) = mpsc::channel::<u8>();
let (dead_tx, dead_rx) = mpsc::channel::<u8>();
subs.insert("live".into(), live_tx);
subs.insert("dead".into(), dead_tx);
drop(dead_rx);
let snapshot: Vec<(String, mpsc::Sender<u8>)> = subs
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
let mut dead = Vec::new();
for (url, tx) in &snapshot {
if tx.send(1).is_err() {
dead.push(url.clone());
}
}
for url in &dead {
subs.remove(url);
}
assert!(subs.contains_key("live"));
assert!(!subs.contains_key("dead"), "dead subscriber must be pruned");
}
#[test]
fn quic_seed_is_stable_and_key_separated() {
assert_eq!(super::quic_seed("x"), super::quic_seed("x"));
assert_ne!(super::quic_seed("x"), super::quic_seed("y"));
}
#[test]
fn derive_quic_identity_is_deterministic() {
let (c1, _k1) = super::derive_quic_identity("shared-secret").expect("derive");
let (c2, _k2) = super::derive_quic_identity("shared-secret").expect("derive");
assert_eq!(
c1.as_ref(),
c2.as_ref(),
"same peer_key must yield byte-identical cert"
);
assert_eq!(super::cert_sha256(&c1), super::cert_sha256(&c2));
}
#[test]
fn derive_quic_identity_differs_by_key() {
let (a, _) = super::derive_quic_identity("key-a").expect("derive a");
let (b, _) = super::derive_quic_identity("key-b").expect("derive b");
assert_ne!(
super::cert_sha256(&a),
super::cert_sha256(&b),
"different peer_key must yield a different cert"
);
}
#[test]
fn derived_key_is_a_valid_private_key() {
let (_c, k) = super::derive_quic_identity("shared-secret").expect("derive");
assert!(matches!(k, rustls::pki_types::PrivateKeyDer::Pkcs8(_)));
}
#[test]
fn pinned_verify_accepts_matching_cert() {
use rustls::client::danger::ServerCertVerifier;
use rustls::pki_types::{ServerName, UnixTime};
let (cert, _k) = super::derive_quic_identity("shared-secret").expect("derive");
let v = super::PinnedVerify {
expected: super::cert_sha256(&cert),
};
let name = ServerName::try_from("localhost").unwrap();
let r = v.verify_server_cert(&cert, &[], &name, &[], UnixTime::now());
assert!(r.is_ok(), "matching cert must verify");
}
#[test]
fn pinned_verify_rejects_mismatched_cert() {
use rustls::client::danger::ServerCertVerifier;
use rustls::pki_types::{ServerName, UnixTime};
let (good, _) = super::derive_quic_identity("shared-secret").expect("derive");
let (attacker, _) = super::derive_quic_identity("attacker-key").expect("derive");
let v = super::PinnedVerify {
expected: super::cert_sha256(&good),
};
let name = ServerName::try_from("localhost").unwrap();
let r = v.verify_server_cert(&attacker, &[], &name, &[], UnixTime::now());
assert!(r.is_err(), "a cert not matching the pin must be rejected");
}
#[test]
fn build_tls_configs_keyed_and_keyless() {
let _ = super::build_tls_configs("shared-secret");
let _ = super::build_tls_configs("");
}
#[test]
fn task_feed_config_keeps_idle_connections_alive() {
let rendered = format!("{:?}", super::task_feed_transport_config());
assert!(
rendered.contains("keep_alive_interval: Some("),
"task feed needs a keepalive or idle subscriptions die: {rendered}"
);
assert!(
rendered.contains("max_idle_timeout: Some("),
"task feed needs an explicit idle timeout so it does not rely on a \
default that may be shorter than the keepalive: {rendered}"
);
}
#[test]
fn resolve_peer_addr_accepts_literal_ips() {
let addr = super::resolve_peer_addr("10.13.13.15", 9001)
.expect("a literal IP must resolve without a DNS lookup");
assert_eq!(addr.to_string(), "10.13.13.15:9001");
}
#[test]
fn resolve_peer_addr_accepts_dns_names() {
let addr = super::resolve_peer_addr("localhost", 9001)
.expect("a DNS name must resolve — parse::<SocketAddr>() cannot do this");
assert_eq!(addr.port(), 9001);
assert!(
addr.ip().is_loopback(),
"localhost should be loopback: {addr}"
);
}
#[test]
fn resolve_peer_addr_prefers_ipv4() {
let addr = super::resolve_peer_addr("localhost", 9001).expect("localhost resolves");
assert!(addr.is_ipv4(), "must prefer the v4 address, got {addr}");
}
#[test]
fn resolve_peer_addr_rejects_unresolvable_names() {
assert!(super::resolve_peer_addr("no-such-peer.invalid", 9001).is_none());
}
}
#[cfg(test)]
mod feed_subscription_tests {
use super::{keep_subscribing, run_feed_subscription, FeedAttempt};
use crate::broker::peer::PeerManager;
use std::cell::{Cell, RefCell};
use std::time::Duration;
const PEER: &str = "http://10.13.13.7:9000";
#[test]
fn a_feed_subscription_keeps_going_only_while_the_peer_advertises_a_quic_port() {
assert!(keep_subscribing(9001));
assert!(!keep_subscribing(0));
}
#[test]
fn the_subscription_loop_ends_once_the_peers_cached_port_reads_0() {
let pm = PeerManager::new(Some("10.13.13.5"), &[], 9000, "k".into(), true);
pm.set_quic_port(PEER, 9001);
let attempts = Cell::new(0u32);
let pauses = RefCell::new(Vec::new());
run_feed_subscription(
|| pm.get_quic_port(PEER),
|| {
attempts.set(attempts.get() + 1);
assert!(attempts.get() < 10, "the loop ignored the cleared port");
if attempts.get() == 1 {
return FeedAttempt::Closed;
}
pm.set_quic_port(PEER, 0);
FeedAttempt::ConnectFailed
},
|d| pauses.borrow_mut().push(d),
);
assert_eq!(attempts.get(), 2);
assert_eq!(
*pauses.borrow(),
vec![Duration::from_secs(2), Duration::from_secs(5)]
);
}
#[test]
fn the_subscription_loop_makes_no_attempt_without_a_cached_port() {
let pm = PeerManager::new(Some("10.13.13.5"), &[], 9000, "k".into(), true);
run_feed_subscription(
|| pm.get_quic_port(PEER),
|| panic!("dialed the task feed of a peer with no QUIC port"),
|_| {},
);
}
}
#[cfg(test)]
mod offer_gate_tests {
use super::{gate_offer, QuicTransport};
use crate::broker::node_identity::NodeKey;
use crate::broker::server::PeerGateFlags;
use crate::broker::task_board::TaskOffer;
use crate::broker::{BrokerConfig, BrokerState};
use std::sync::Arc;
const ARMED: PeerGateFlags = PeerGateFlags {
enforce_node_sig: true,
require_voucher: true,
};
const SIG_ONLY: PeerGateFlags = PeerGateFlags {
enforce_node_sig: true,
require_voucher: false,
};
const VOUCHER_ONLY: PeerGateFlags = PeerGateFlags {
enforce_node_sig: false,
require_voucher: true,
};
const UNSET: PeerGateFlags = PeerGateFlags {
enforce_node_sig: false,
require_voucher: false,
};
const OLD_OFFER: &str = r#"{"task_id":"t1","payload_b64":"","max_price_per_hour":100.0,"estimated_duration_secs":1.0,"timeout_secs":10.0,"requester_user_id":"u","source_broker":"n","peer_key":"k"}"#;
fn receiver(sender: &NodeKey) -> (Arc<BrokerState>, NodeKey) {
let st = BrokerState::with_config(BrokerConfig {
api_url: Some("http://127.0.0.1:1".into()),
..Default::default()
});
st.node_roster.insert(sender.public_b64(), false);
let hub = NodeKey::generate();
*st.dash_voucher_pubkey.write().unwrap() = vec![hub.public_b64()];
(Arc::new(st), hub)
}
fn old_offer() -> TaskOffer {
serde_json::from_str(OLD_OFFER).unwrap()
}
fn vouched_offer(hub: &NodeKey) -> TaskOffer {
let signed = r#"{"v":1,"requester_user":"u","budget_credits":1000000.0,"task_nonce":"n","exp":9999999999}"#;
TaskOffer {
voucher_signed_json: signed.to_string(),
voucher_sig: hub.sign(signed.as_bytes()),
..old_offer()
}
}
fn send(offer: &TaskOffer, key: &NodeKey) -> Vec<u8> {
let mut o = offer.clone();
o.sign_for_quic(key);
serde_json::to_vec(&o).unwrap()
}
fn receive(st: &Arc<BrokerState>, raw: &[u8], flags: PeerGateFlags) -> Result<(), String> {
let offer: TaskOffer = serde_json::from_slice(raw).unwrap();
match gate_offer(st, raw, &offer, flags) {
None => Ok(()),
Some(rej) => {
assert_eq!(rej.task_id, "t1");
Err(rej.reason)
}
}
}
#[test]
fn enforced_node_sig_refuses_an_unsigned_offer() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let err = receive(&st, OLD_OFFER.as_bytes(), SIG_ONLY).unwrap_err();
assert!(err.contains("node signature check failed"), "{err}");
}
#[test]
fn enforced_node_sig_refuses_a_tampered_offer() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let raw = send(&old_offer(), &sender);
let mut v: serde_json::Value = serde_json::from_slice(&raw).unwrap();
v["max_price_per_hour"] = serde_json::json!(1.0e9);
let tampered = serde_json::to_vec(&v).unwrap();
let err = receive(&st, &tampered, SIG_ONLY).unwrap_err();
assert!(err.contains("bad signature"), "{err}");
}
#[test]
fn enforced_node_sig_refuses_an_unrostered_signer() {
let (st, _) = receiver(&NodeKey::generate());
let raw = send(&old_offer(), &NodeKey::generate());
let err = receive(&st, &raw, SIG_ONLY).unwrap_err();
assert!(err.contains("not in roster"), "{err}");
}
#[test]
fn enforced_node_sig_refuses_a_replayed_offer() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let raw = send(&old_offer(), &sender);
assert_eq!(receive(&st, &raw, SIG_ONLY), Ok(()));
let err = receive(&st, &raw, SIG_ONLY).unwrap_err();
assert!(err.contains("replayed"), "{err}");
}
#[test]
fn an_http_signature_does_not_pass_as_a_quic_one() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let mut offer = old_offer();
offer.sign_for_quic(&sender);
let raw = serde_json::to_vec(&offer).unwrap();
let body = crate::broker::task_board::quic_offer_sign_bytes(&raw).unwrap();
let hdrs = sender.sign_headers("POST", "/peer/tasks/offer", &body);
let sig = hdrs.iter().find(|(k, _)| k == "X-Node-Sig").unwrap();
offer.node_sig.as_mut().unwrap().sig = sig.1.clone();
let raw = serde_json::to_vec(&offer).unwrap();
let err = receive(&st, &raw, SIG_ONLY).unwrap_err();
assert!(err.contains("bad signature"), "{err}");
}
#[test]
fn required_voucher_refuses_a_foreign_offer_without_one() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let raw = send(&old_offer(), &sender);
let err = receive(&st, &raw, VOUCHER_ONLY).unwrap_err();
assert!(err.contains("voucher check failed"), "{err}");
}
#[test]
fn unset_flags_accept_old_and_badly_signed_offers() {
let (st, _) = receiver(&NodeKey::generate());
assert_eq!(receive(&st, OLD_OFFER.as_bytes(), UNSET), Ok(()));
let raw = send(&old_offer(), &NodeKey::generate());
assert_eq!(receive(&st, &raw, UNSET), Ok(()));
}
#[test]
fn an_upgraded_sender_passes_an_armed_receiver() {
let sender = NodeKey::generate();
let (st, hub) = receiver(&sender);
let raw = send(&vouched_offer(&hub), &sender);
assert_eq!(receive(&st, &raw, ARMED), Ok(()));
}
#[test]
fn the_signature_covers_unknown_fields_and_ignores_key_order() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let mut v: serde_json::Value = serde_json::from_slice(OLD_OFFER.as_bytes()).unwrap();
v["from_a_newer_build"] = serde_json::json!({"b": 2, "a": 1});
let body =
crate::broker::task_board::quic_offer_sign_bytes(&serde_json::to_vec(&v).unwrap())
.unwrap();
let hdrs = sender.sign_headers(
"POST",
crate::broker::task_board::QUIC_OFFER_SIGN_PATH,
&body,
);
let h = |n: &str| hdrs.iter().find(|(k, _)| k == n).unwrap().1.clone();
v["node_sig"] = serde_json::json!({
"node_id": h("X-Node-Id"), "sig": h("X-Node-Sig"),
"nonce": h("X-Node-Nonce"), "ts": h("X-Node-Ts").parse::<u64>().unwrap(),
});
let reordered = format!(
r#"{{"from_a_newer_build":{{"a":1,"b":2}},"node_sig":{},{}"#,
v["node_sig"],
&OLD_OFFER[1..]
);
assert_eq!(receive(&st, reordered.as_bytes(), SIG_ONLY), Ok(()));
let changed = reordered.replace(r#""a":1"#, r#""a":9"#);
assert!(receive(&st, changed.as_bytes(), SIG_ONLY).is_err());
}
#[test]
fn quic_offers_from_an_upgraded_sender_pass_an_armed_receiver_on_the_wire() {
let a = Arc::new(BrokerState::new());
let (b, hub) = receiver(&a.node_key);
let lo = "127.0.0.1:0".parse().unwrap();
let qa = QuicTransport::new(lo, a.clone()).unwrap();
let qb = QuicTransport::new(lo, b.clone()).unwrap();
qb.set_gate_flags(ARMED);
let addr_b = qb.local_addr().unwrap();
let err = qa.offer_task(addr_b, &vouched_offer(&hub)).unwrap_err();
assert!(err.contains("No matching local worker"), "{err}");
b.node_roster.clear();
let err = qa.offer_task(addr_b, &vouched_offer(&hub)).unwrap_err();
assert!(err.contains("node signature check failed"), "{err}");
}
use crate::broker::task_board::{TaskAccept, TaskReject, TaskResult};
use std::sync::atomic::{AtomicUsize, Ordering};
fn counting_server(count_path: &'static str, body: &'static str) -> (Arc<AtomicUsize>, String) {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
let hits = Arc::new(AtomicUsize::new(0));
let h = hits.clone();
std::thread::spawn(move || {
for req in server.incoming_requests() {
if req.url().contains(count_path) {
h.fetch_add(1, Ordering::SeqCst);
}
let _ = req.respond(tiny_http::Response::from_string(body));
}
});
(hits, format!("http://127.0.0.1:{port}"))
}
fn working_receiver(
sender: &NodeKey,
) -> (
Arc<BrokerState>,
NodeKey,
Arc<AtomicUsize>,
Arc<AtomicUsize>,
) {
let (jobs, worker_url) = counting_server("/execute", "ok");
let (redeems, hub_url) = counting_server("/voucher/redeem", r#"{"redeemed":true}"#);
let st = BrokerState::with_config(BrokerConfig {
api_url: Some(hub_url),
api_key: Some("zk_test".into()),
..Default::default()
});
st.set_local_mode(true);
st.node_roster.insert(sender.public_b64(), false);
let hub = NodeKey::generate();
*st.dash_voucher_pubkey.write().unwrap() = vec![hub.public_b64()];
st.workers
.register(crate::broker::worker::test_registration(
"w0", worker_url, 1.0,
));
(Arc::new(st), hub, jobs, redeems)
}
fn wire(
flags: PeerGateFlags,
) -> (
Arc<QuicTransport>,
std::net::SocketAddr,
Arc<BrokerState>,
NodeKey,
Arc<AtomicUsize>,
Arc<AtomicUsize>,
) {
let a = Arc::new(BrokerState::new());
let (b, hub, jobs, redeems) = working_receiver(&a.node_key);
let lo = "127.0.0.1:0".parse().unwrap();
let qa = QuicTransport::new(lo, a).unwrap();
let qb = QuicTransport::new(lo, b.clone()).unwrap();
qb.set_gate_flags(flags);
let addr = qb.local_addr().unwrap();
std::mem::forget(qb);
(qa, addr, b, hub, jobs, redeems)
}
fn send_raw(qa: &QuicTransport, addr: std::net::SocketAddr, offer: &TaskOffer) -> Vec<u8> {
qa.runtime.block_on(async {
let conn = qa.get_or_connect(addr).await.unwrap();
let (mut s, mut r) = conn.open_bi().await.unwrap();
let mut o = offer.clone();
o.peer_key = qa.peer_key.clone();
o.sign_for_quic(&qa.node_key);
super::write_msg(&mut s, &serde_json::to_vec(&o).unwrap())
.await
.unwrap();
super::read_msg(&mut r).await.unwrap()
})
}
#[test]
fn a_vouched_one_shot_quic_offer_redeems_once_and_a_replay_is_refused() {
let (qa, addr, _b, hub, jobs, redeems) = wire(ARMED);
let offer = vouched_offer(&hub);
let res = qa.offer_task(addr, &offer).unwrap();
assert_eq!(res.task_id, "t1");
assert!(
res.worker_uri.starts_with("zc://worker-"),
"{}",
res.worker_uri
);
assert_eq!(jobs.load(Ordering::SeqCst), 1);
assert_eq!(redeems.load(Ordering::SeqCst), 1, "one run, one redeem");
let err = qa.offer_task(addr, &offer).unwrap_err();
assert!(err.contains("voucher already used"), "{err}");
assert_eq!(jobs.load(Ordering::SeqCst), 1, "a replay must not run");
assert_eq!(
redeems.load(Ordering::SeqCst),
1,
"a replay must not redeem"
);
}
#[test]
fn a_failed_run_releases_the_voucher_for_a_retry() {
let (qa, addr, b, hub, _jobs, redeems) = wire(ARMED);
let mut bad = vouched_offer(&hub);
bad.payload_b64 = "%%% not base64".into();
assert!(qa.offer_task(addr, &bad).is_err());
assert!(b.claimed_voucher_nonces.is_empty());
assert_eq!(redeems.load(Ordering::SeqCst), 0);
qa.offer_task(addr, &vouched_offer(&hub)).unwrap();
assert_eq!(redeems.load(Ordering::SeqCst), 1);
}
#[test]
fn a_quic_offer_pinned_to_another_node_is_refused() {
let (qa, addr, b, _hub, jobs, _) = wire(UNSET);
let mut offer = old_offer();
offer.target_node = "zc://node-0000000000000000".into();
let err = qa.offer_task(addr, &offer).unwrap_err();
assert!(err.contains("pinned to node"), "{err}");
assert_eq!(jobs.load(Ordering::SeqCst), 0);
offer.target_node = b.node_key.node_uri();
qa.offer_task(addr, &offer).unwrap();
assert_eq!(jobs.load(Ordering::SeqCst), 1);
}
#[test]
fn a_two_phase_quic_offer_reserves_like_http() {
let (qa, addr, b, hub, jobs, redeems) = wire(ARMED);
let offer = TaskOffer {
two_phase: true,
..vouched_offer(&hub)
};
let reply = send_raw(&qa, addr, &offer);
let accept: TaskAccept = serde_json::from_slice(&reply).unwrap();
assert_eq!(accept.task_id, "t1");
assert!(serde_json::from_slice::<TaskResult>(&reply).is_err());
assert!(b.pending_offers.contains_key("t1"));
assert_eq!(jobs.load(Ordering::SeqCst), 0, "a reserve must not run");
let reply = send_raw(
&qa,
addr,
&TaskOffer {
task_id: "t2".into(),
..offer.clone()
},
);
let rej: TaskReject = serde_json::from_slice(&reply).unwrap();
assert!(
rej.reason.contains("voucher already used"),
"{}",
rej.reason
);
let pending = crate::broker::server::take_committed(&b, "t1").unwrap();
assert_eq!(pending.offer.task_id, "t1");
assert_eq!(redeems.load(Ordering::SeqCst), 0);
}
#[test]
fn cancelling_a_two_phase_quic_reservation_releases_its_voucher() {
let (qa, addr, b, hub, jobs, _) = wire(ARMED);
let offer = TaskOffer {
two_phase: true,
..vouched_offer(&hub)
};
let _: TaskAccept = serde_json::from_slice(&send_raw(&qa, addr, &offer)).unwrap();
crate::broker::server::cancel_offer(&b, "t1");
assert!(b.claimed_voucher_nonces.is_empty());
qa.offer_task(addr, &vouched_offer(&hub)).unwrap();
assert_eq!(jobs.load(Ordering::SeqCst), 1);
}
}