use std::net::{SocketAddr, ToSocketAddrs};
use std::sync::{Arc, Mutex, OnceLock};
use dashmap::DashMap;
use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName, UnixTime};
use tokio::runtime::Runtime;
use std::collections::HashSet;
use std::sync::mpsc;
use std::thread;
use std::time::Duration;
use super::node_identity::NodeKey;
use super::server::{
quic_offer_gate_from, quic_subscribe_gate, run_peer_offer_from, PeerGateFlags, PeerOfferOutcome,
};
use super::task_board::{SubscribeMessage, TaskOffer, TaskReject, TaskResult};
use super::BrokerState;
type ExecuteFn = Arc<
dyn Fn(&BrokerState, &TaskOffer, bool, Option<&str>) -> Result<TaskResult, TaskReject>
+ Send
+ Sync,
>;
type OfferReceiver = mpsc::Receiver<(
TaskOffer,
mpsc::Sender<(String, Result<TaskResult, TaskReject>)>,
)>;
fn quic_seed(peer_key: &str) -> [u8; 32] {
let mut input = Vec::with_capacity(12 + peer_key.len());
input.extend_from_slice(b"zc-quic-pin\0");
input.extend_from_slice(peer_key.as_bytes());
let digest = ring::digest::digest(&ring::digest::SHA256, &input);
let mut out = [0u8; 32];
out.copy_from_slice(digest.as_ref());
out
}
fn ed25519_pkcs8(seed: &[u8; 32]) -> Vec<u8> {
const PREFIX: [u8; 16] = [
0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04,
0x20,
];
let mut der = Vec::with_capacity(48);
der.extend_from_slice(&PREFIX);
der.extend_from_slice(seed);
der
}
fn derive_quic_identity(
peer_key: &str,
) -> Result<(CertificateDer<'static>, PrivateKeyDer<'static>), String> {
let seed = quic_seed(peer_key);
let pkcs8 = ed25519_pkcs8(&seed);
let pkcs8_der = PrivatePkcs8KeyDer::from(pkcs8.clone());
let key_pair = rcgen::KeyPair::from_pkcs8_der_and_sign_algo(&pkcs8_der, &rcgen::PKCS_ED25519)
.map_err(|e| format!("derive keypair: {}", e))?;
let mut params = rcgen::CertificateParams::new(vec!["zakuro-broker".to_string()])
.map_err(|e| format!("cert params: {}", e))?;
params.serial_number = Some(rcgen::SerialNumber::from_slice(&[0x01]));
params.not_before = rcgen::date_time_ymd(2020, 1, 1);
params.not_after = rcgen::date_time_ymd(2100, 1, 1);
params.distinguished_name = {
let mut dn = rcgen::DistinguishedName::new();
dn.push(rcgen::DnType::CommonName, "zakuro-broker");
dn
};
let cert = params
.self_signed(&key_pair)
.map_err(|e| format!("self-sign: {}", e))?;
let cert_der = CertificateDer::from(cert.der().to_vec());
let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(pkcs8));
Ok((cert_der, key))
}
fn cert_sha256(cert: &CertificateDer) -> [u8; 32] {
let digest = ring::digest::digest(&ring::digest::SHA256, cert.as_ref());
let mut out = [0u8; 32];
out.copy_from_slice(digest.as_ref());
out
}
fn make_server_config(
node: Option<(CertificateDer<'static>, PrivateKeyDer<'static>)>,
legacy: Option<(CertificateDer<'static>, PrivateKeyDer<'static>)>,
) -> Result<quinn::ServerConfig, String> {
let provider = rustls::crypto::ring::default_provider();
let certified = |(cert, key): (CertificateDer<'static>, PrivateKeyDer<'static>)| {
provider
.key_provider
.load_private_key(key)
.map(|k| Arc::new(rustls::sign::CertifiedKey::new(vec![cert], k)))
.map_err(|e| format!("load TLS key: {e}"))
};
let mut alpn = Vec::new();
let v2 = match node {
Some(n) => {
alpn.push(ALPN_V2.to_vec());
Some(certified(n)?)
}
None => None,
};
let legacy = match legacy {
Some(l) => {
alpn.push(ALPN_LEGACY.to_vec());
Some(certified(l)?)
}
None => None,
};
if alpn.is_empty() {
return Err("no QUIC protocol available".into());
}
let mut sc = rustls::ServerConfig::builder()
.with_client_cert_verifier(Arc::new(NodeClientVerifier))
.with_cert_resolver(Arc::new(AlpnCertResolver { v2, legacy }));
sc.alpn_protocols = alpn;
let quic_sc = quinn::crypto::rustls::QuicServerConfig::try_from(sc)
.map_err(|e| format!("QUIC requires a rustls config supporting TLS 1.3: {e}"))?;
let mut cfg = quinn::ServerConfig::with_crypto(Arc::new(quic_sc));
cfg.transport_config(Arc::new(task_feed_transport_config()));
Ok(cfg)
}
fn task_feed_transport_config() -> quinn::TransportConfig {
let mut tc = quinn::TransportConfig::default();
tc.keep_alive_interval(Some(Duration::from_secs(5)));
tc.max_idle_timeout(Some(
Duration::from_secs(30)
.try_into()
.expect("30s is a valid QUIC idle timeout"),
));
tc
}
fn resolve_peer_addr(host: &str, port: u16) -> Option<SocketAddr> {
let candidates: Vec<SocketAddr> = (host, port).to_socket_addrs().ok()?.collect();
candidates
.iter()
.find(|a| a.is_ipv4())
.or_else(|| candidates.first())
.copied()
}
fn quinn_client(cc: rustls::ClientConfig) -> quinn::ClientConfig {
let quic_cc = quinn::crypto::rustls::QuicClientConfig::try_from(cc)
.expect("QUIC requires a rustls config supporting TLS 1.3");
let mut cfg = quinn::ClientConfig::new(Arc::new(quic_cc));
cfg.transport_config(Arc::new(task_feed_transport_config()));
cfg
}
fn make_legacy_client_config(pin: [u8; 32]) -> quinn::ClientConfig {
let mut cc = rustls::ClientConfig::builder()
.dangerous()
.with_custom_certificate_verifier(Arc::new(PinnedVerify { expected: pin }))
.with_no_client_auth();
cc.alpn_protocols = vec![ALPN_LEGACY.to_vec()];
quinn_client(cc)
}
fn build_legacy(
peer_key: &str,
derive: impl Fn(&str) -> Result<(CertificateDer<'static>, PrivateKeyDer<'static>), String>,
) -> Result<
(
CertificateDer<'static>,
PrivateKeyDer<'static>,
quinn::ClientConfig,
),
String,
> {
if peer_key.is_empty() {
return Err("no ZAKURO_PEER_KEY: the legacy zk-quic protocol needs a pinned cert".into());
}
let (cert, key) =
derive(peer_key).map_err(|e| format!("peer-key cert derivation failed ({e})"))?;
let client = make_legacy_client_config(cert_sha256(&cert));
Ok((cert, key, client))
}
#[derive(Debug)]
struct PinnedVerify {
expected: [u8; 32],
}
impl rustls::client::danger::ServerCertVerifier for PinnedVerify {
fn verify_server_cert(
&self,
end_entity: &CertificateDer<'_>,
_intermediates: &[CertificateDer<'_>],
_server_name: &ServerName<'_>,
_ocsp_response: &[u8],
_now: UnixTime,
) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
if cert_sha256(end_entity) == self.expected {
Ok(rustls::client::danger::ServerCertVerified::assertion())
} else {
Err(rustls::Error::InvalidCertificate(
rustls::CertificateError::ApplicationVerificationFailure,
))
}
}
fn verify_tls12_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
}
fn verify_tls13_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
}
fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
rustls::crypto::ring::default_provider()
.signature_verification_algorithms
.supported_schemes()
}
}
const ALPN_LEGACY: &[u8] = b"zk-quic";
const ALPN_V2: &[u8] = b"zk-quic/2";
const V2_REFUSED: u32 = 0x7a01;
const V2_COOLDOWN: Duration = Duration::from_secs(60);
const NO_APPLICATION_PROTOCOL: u64 = 0x100 + 120;
fn legacy_allowed(value: Option<&str>) -> bool {
!matches!(value, Some("0") | Some("false"))
}
const ED25519_SPKI_PREFIX: [u8; 12] = [
0x30, 0x2a, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x03, 0x21, 0x00,
];
fn cert_ed25519_pubkey(cert: &[u8]) -> Option<[u8; 32]> {
cert.windows(ED25519_SPKI_PREFIX.len() + 32)
.find(|w| w[..ED25519_SPKI_PREFIX.len()] == ED25519_SPKI_PREFIX)
.and_then(|w| w[ED25519_SPKI_PREFIX.len()..].try_into().ok())
}
fn pubkey_fingerprint(raw: &[u8]) -> String {
let d = ring::digest::digest(&ring::digest::SHA256, raw);
d.as_ref()[..8].iter().map(|b| format!("{b:02x}")).collect()
}
fn node_cert(key: &NodeKey) -> Result<CertificateDer<'static>, String> {
let pkcs8 = PrivatePkcs8KeyDer::from(key.pkcs8_bytes().to_vec());
let kp = rcgen::KeyPair::from_pkcs8_der_and_sign_algo(&pkcs8, &rcgen::PKCS_ED25519)
.map_err(|e| format!("node keypair: {e}"))?;
let mut params = rcgen::CertificateParams::new(vec!["zakuro-node".to_string()])
.map_err(|e| format!("cert params: {e}"))?;
params.not_before = rcgen::date_time_ymd(2020, 1, 1);
params.not_after = rcgen::date_time_ymd(2100, 1, 1);
let cert = params
.self_signed(&kp)
.map_err(|e| format!("self-sign: {e}"))?;
Ok(CertificateDer::from(cert.der().to_vec()))
}
fn node_tls_identity(
key: &NodeKey,
) -> Result<(CertificateDer<'static>, PrivateKeyDer<'static>), String> {
Ok((
node_cert(key)?,
PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(key.pkcs8_bytes().to_vec())),
))
}
fn verify_ed25519_handshake(
cert: &CertificateDer<'_>,
message: &[u8],
dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
if dss.scheme != rustls::SignatureScheme::ED25519 {
return Err(rustls::Error::General(
"zk-quic/2 needs an Ed25519 node key".into(),
));
}
let pk = cert_ed25519_pubkey(cert.as_ref())
.ok_or_else(|| rustls::Error::InvalidCertificate(rustls::CertificateError::BadEncoding))?;
ring::signature::UnparsedPublicKey::new(&ring::signature::ED25519, pk)
.verify(message, dss.signature())
.map_err(|_| rustls::Error::InvalidCertificate(rustls::CertificateError::BadSignature))?;
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
}
#[derive(Debug)]
struct AlpnCertResolver {
v2: Option<Arc<rustls::sign::CertifiedKey>>,
legacy: Option<Arc<rustls::sign::CertifiedKey>>,
}
impl rustls::server::ResolvesServerCert for AlpnCertResolver {
fn resolve(
&self,
hello: rustls::server::ClientHello<'_>,
) -> Option<Arc<rustls::sign::CertifiedKey>> {
let offers_v2 = hello
.alpn()
.map(|mut it| it.any(|p| p == ALPN_V2))
.unwrap_or(false);
if offers_v2 && self.v2.is_some() {
self.v2.clone()
} else {
self.legacy.clone()
}
}
}
#[derive(Debug)]
struct NodeClientVerifier;
impl rustls::server::danger::ClientCertVerifier for NodeClientVerifier {
fn offer_client_auth(&self) -> bool {
true
}
fn client_auth_mandatory(&self) -> bool {
false
}
fn root_hint_subjects(&self) -> &[rustls::DistinguishedName] {
&[]
}
fn verify_client_cert(
&self,
end_entity: &CertificateDer<'_>,
_intermediates: &[CertificateDer<'_>],
_now: UnixTime,
) -> Result<rustls::server::danger::ClientCertVerified, rustls::Error> {
cert_ed25519_pubkey(end_entity.as_ref())
.map(|_| rustls::server::danger::ClientCertVerified::assertion())
.ok_or(rustls::Error::InvalidCertificate(
rustls::CertificateError::BadEncoding,
))
}
fn verify_tls12_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
Err(rustls::Error::General("QUIC is TLS 1.3 only".into()))
}
fn verify_tls13_signature(
&self,
message: &[u8],
cert: &CertificateDer<'_>,
dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
verify_ed25519_handshake(cert, message, dss)
}
fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
vec![rustls::SignatureScheme::ED25519]
}
}
#[derive(Debug)]
struct NodeServerVerifier {
expected_fingerprint: String,
}
impl rustls::client::danger::ServerCertVerifier for NodeServerVerifier {
fn verify_server_cert(
&self,
end_entity: &CertificateDer<'_>,
_intermediates: &[CertificateDer<'_>],
_server_name: &ServerName<'_>,
_ocsp_response: &[u8],
_now: UnixTime,
) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
match cert_ed25519_pubkey(end_entity.as_ref()) {
Some(pk) if pubkey_fingerprint(&pk) == self.expected_fingerprint => {
Ok(rustls::client::danger::ServerCertVerified::assertion())
}
_ => Err(rustls::Error::InvalidCertificate(
rustls::CertificateError::ApplicationVerificationFailure,
)),
}
}
fn verify_tls12_signature(
&self,
_message: &[u8],
_cert: &CertificateDer<'_>,
_dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
Err(rustls::Error::General("QUIC is TLS 1.3 only".into()))
}
fn verify_tls13_signature(
&self,
message: &[u8],
cert: &CertificateDer<'_>,
dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
verify_ed25519_handshake(cert, message, dss)
}
fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
vec![rustls::SignatureScheme::ED25519]
}
}
fn make_v2_client_config(
cert: CertificateDer<'static>,
pkcs8: &[u8],
expected_fingerprint: String,
) -> Result<quinn::ClientConfig, String> {
let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(pkcs8.to_vec()));
let mut cc = rustls::ClientConfig::builder()
.dangerous()
.with_custom_certificate_verifier(Arc::new(NodeServerVerifier {
expected_fingerprint,
}))
.with_client_auth_cert(vec![cert], key)
.map_err(|e| format!("client cert: {e}"))?;
cc.alpn_protocols = vec![ALPN_V2.to_vec()];
Ok(quinn_client(cc))
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct VerifiedPeer {
pubkey_b64: String,
fingerprint: String,
}
fn conn_alpn(conn: &quinn::Connection) -> Option<Vec<u8>> {
conn.handshake_data()?
.downcast::<quinn::crypto::rustls::HandshakeData>()
.ok()?
.protocol
.clone()
}
fn conn_peer_cert(conn: &quinn::Connection) -> Option<CertificateDer<'static>> {
conn.peer_identity()?
.downcast::<Vec<CertificateDer<'static>>>()
.ok()?
.into_iter()
.next()
}
fn proto_name(conn: &quinn::Connection) -> &'static str {
match conn_alpn(conn).as_deref() {
Some(a) if a == ALPN_V2 => "zk-quic/2",
Some(a) if a == ALPN_LEGACY => "zk-quic",
_ => "unknown",
}
}
fn verified_peer(state: &BrokerState, conn: &quinn::Connection) -> Result<VerifiedPeer, String> {
let cert = conn_peer_cert(conn).ok_or("client certificate required")?;
let pk = cert_ed25519_pubkey(cert.as_ref()).ok_or("client certificate has no Ed25519 key")?;
rostered_peer(state, &pk)
}
fn rostered_peer(state: &BrokerState, pk: &[u8; 32]) -> Result<VerifiedPeer, String> {
use base64::Engine;
let pubkey_b64 = base64::engine::general_purpose::STANDARD.encode(pk);
match state.node_roster.get(&pubkey_b64).map(|r| *r.value()) {
Some(false) => Ok(VerifiedPeer {
fingerprint: pubkey_fingerprint(pk),
pubkey_b64,
}),
Some(true) => Err("node key is revoked".into()),
None => Err("node key is not in the roster".into()),
}
}
fn flags_for_conn(
base: PeerGateFlags,
peer: Option<&VerifiedPeer>,
node_sig: Option<&crate::broker::task_board::OfferNodeSig>,
) -> Result<PeerGateFlags, String> {
match (peer, node_sig) {
(None, _) => Ok(base),
(Some(_), None) => Ok(PeerGateFlags {
enforce_node_sig: false,
..base
}),
(Some(p), Some(sig)) if sig.node_id == p.pubkey_b64 => Ok(PeerGateFlags {
enforce_node_sig: true,
..base
}),
(Some(_), Some(_)) => {
Err("node signature signer does not match the connection's node key".into())
}
}
}
fn conn_peer_key_ok(expected: &str, provided: &str, verified: bool) -> bool {
if verified && expected.is_empty() {
return true;
}
peer_key_ok(expected, provided)
}
async fn write_msg(send: &mut quinn::SendStream, payload: &[u8]) -> Result<(), String> {
let len = (payload.len() as u32).to_be_bytes();
send.write_all(&len).await.map_err(|e| e.to_string())?;
send.write_all(payload).await.map_err(|e| e.to_string())?;
send.finish().map_err(|e| e.to_string())?;
Ok(())
}
async fn write_msg_keep_open(send: &mut quinn::SendStream, payload: &[u8]) -> Result<(), String> {
let len = (payload.len() as u32).to_be_bytes();
send.write_all(&len).await.map_err(|e| e.to_string())?;
send.write_all(payload).await.map_err(|e| e.to_string())?;
Ok(())
}
const MAX_QUIC_MSG_BYTES: usize = 8 * 1024 * 1024;
fn cap_msg_len(len_bytes: [u8; 4]) -> Result<usize, String> {
let len = u32::from_be_bytes(len_bytes) as usize;
if len > MAX_QUIC_MSG_BYTES {
return Err(format!(
"message too large: {} bytes (max {})",
len, MAX_QUIC_MSG_BYTES
));
}
Ok(len)
}
async fn read_msg(recv: &mut quinn::RecvStream) -> Result<Vec<u8>, String> {
let mut len_buf = [0u8; 4];
recv.read_exact(&mut len_buf)
.await
.map_err(|e| e.to_string())?;
let len = cap_msg_len(len_buf)?;
const CHUNK: usize = 64 * 1024;
let mut buf: Vec<u8> = Vec::with_capacity(len.min(CHUNK));
let mut remaining = len;
let mut chunk = [0u8; CHUNK];
while remaining > 0 {
let want = remaining.min(CHUNK);
recv.read_exact(&mut chunk[..want])
.await
.map_err(|e| e.to_string())?;
buf.extend_from_slice(&chunk[..want]);
remaining -= want;
}
Ok(buf)
}
fn peer_key_ok(expected: &str, provided: &str) -> bool {
!expected.is_empty() && expected == provided
}
type SubscriberOfferTx = mpsc::Sender<(
TaskOffer,
mpsc::Sender<(String, Result<TaskResult, TaskReject>)>,
)>;
pub struct QuicTransport {
runtime: Runtime,
endpoint: quinn::Endpoint,
connections: DashMap<SocketAddr, quinn::Connection>,
subscriber_txs: DashMap<String, SubscriberOfferTx>,
peer_key: String,
node_key: Arc<NodeKey>,
gate_flags_override: OnceLock<PeerGateFlags>,
legacy_client: Option<quinn::ClientConfig>,
node_cert: CertificateDer<'static>,
state: std::sync::Weak<BrokerState>,
fingerprint_hints: DashMap<SocketAddr, String>,
v2_cooldown: DashMap<SocketAddr, std::time::Instant>,
}
pub struct SubFeed {
pub send: quinn::SendStream,
pub recv: quinn::RecvStream,
conn: quinn::Connection,
peer: Option<VerifiedPeer>,
}
enum V2Dial {
Unsupported,
Failed(String),
}
fn is_v2_refused(conn: &quinn::Connection) -> bool {
matches!(
conn.close_reason(),
Some(quinn::ConnectionError::ApplicationClosed(c)) if u64::from(c.error_code) == V2_REFUSED as u64
)
}
impl QuicTransport {
pub fn new(bind_addr: SocketAddr, state: Arc<BrokerState>) -> Result<Arc<Self>, String> {
let legacy = std::env::var("ZAKURO_QUIC_LEGACY").ok();
Self::start(bind_addr, state, legacy_allowed(legacy.as_deref()))
}
fn start(
bind_addr: SocketAddr,
state: Arc<BrokerState>,
legacy_ok: bool,
) -> Result<Arc<Self>, String> {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.map_err(|e| format!("tokio runtime: {}", e))?;
let peer_key = state.peer_manager.peer_key().to_string();
let node_key = state.node_key.clone();
let node = node_tls_identity(&node_key)?;
let node_cert = node.0.clone();
let (legacy_identity, legacy_client) = if legacy_ok {
match build_legacy(&peer_key, derive_quic_identity) {
Ok((c, k, client)) => (Some((c, k)), Some(client)),
Err(e) => {
eprintln!(" [QUIC] legacy zk-quic off: {e}");
(None, None)
}
}
} else {
eprintln!(" [QUIC] legacy zk-quic off (ZAKURO_QUIC_LEGACY)");
(None, None)
};
let server_cfg = make_server_config(Some(node), legacy_identity)?;
let endpoint = runtime.block_on(async {
quinn::Endpoint::server(server_cfg, bind_addr)
.map_err(|e| format!("QUIC bind {}: {}", bind_addr, e))
})?;
let transport = Arc::new(Self {
runtime,
endpoint,
connections: DashMap::new(),
subscriber_txs: DashMap::new(),
peer_key,
node_key,
gate_flags_override: OnceLock::new(),
legacy_client,
node_cert,
state: Arc::downgrade(&state),
fingerprint_hints: DashMap::new(),
v2_cooldown: DashMap::new(),
});
let t = transport.clone();
transport.runtime.spawn(async move {
accept_loop(t, state).await;
});
Ok(transport)
}
fn gate_flags(&self, state: &Arc<BrokerState>) -> PeerGateFlags {
self.gate_flags_override
.get()
.copied()
.unwrap_or_else(|| PeerGateFlags::from_state(state))
}
#[cfg(test)]
fn set_gate_flags(&self, flags: PeerGateFlags) {
let _ = self.gate_flags_override.set(flags);
}
pub fn offer_task(
&self,
peer_addr: SocketAddr,
offer: &TaskOffer,
) -> Result<TaskResult, String> {
self.runtime
.block_on(self.offer_task_inner(peer_addr, offer))
}
async fn offer_task_inner(
&self,
peer_addr: SocketAddr,
offer: &TaskOffer,
) -> Result<TaskResult, String> {
match self.offer_once(peer_addr, offer).await {
Err(_) if self.note_v2_refused(peer_addr) => self.offer_once(peer_addr, offer).await,
r => r,
}
}
async fn offer_once(
&self,
peer_addr: SocketAddr,
offer: &TaskOffer,
) -> Result<TaskResult, String> {
let conn = self.get_or_connect(peer_addr).await?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| format!("open stream: {}", e))?;
let mut offer = offer.clone();
offer.peer_key = self.peer_key.clone();
offer.sign_for_quic(&self.node_key);
let payload = serde_json::to_vec(&offer).map_err(|e| e.to_string())?;
write_msg(&mut send, &payload).await?;
let resp = read_msg(&mut recv).await?;
if let Ok(result) = serde_json::from_slice::<TaskResult>(&resp) {
return Ok(result);
}
if let Ok(reject) = serde_json::from_slice::<TaskReject>(&resp) {
return Err(format!("peer rejected: {}", reject.reason));
}
Err("invalid peer response".into())
}
async fn get_or_connect(&self, addr: SocketAddr) -> Result<quinn::Connection, String> {
if let Some(c) = self.connections.get(&addr) {
if c.close_reason().is_none() {
return Ok(c.clone());
}
}
self.connections.remove(&addr);
let conn = self.dial(addr).await?;
self.connections.insert(addr, conn.clone());
Ok(conn)
}
fn expected_fingerprint(&self, addr: SocketAddr) -> Option<String> {
if let Some(fp) = self.fingerprint_hints.get(&addr) {
return Some(fp.clone());
}
let state = self.state.upgrade()?;
state.peer_manager.peer_urls().into_iter().find_map(|url| {
let port = state.peer_manager.get_quic_port(&url);
let host = url
.trim_start_matches("http://")
.trim_start_matches("https://")
.split(':')
.next()?;
let host = if host.starts_with("127.") {
"127.0.0.1"
} else {
host
};
if port == 0 || resolve_peer_addr(host, port)? != addr {
return None;
}
state.peer_manager.peer_fingerprint(&url)
})
}
async fn dial(&self, addr: SocketAddr) -> Result<quinn::Connection, String> {
let cooling = self
.v2_cooldown
.get(&addr)
.map(|t| *t > std::time::Instant::now())
.unwrap_or(false);
let expected = self.expected_fingerprint(addr);
if let (Some(fp), false) = (&expected, cooling) {
match self.dial_v2(addr, fp).await {
Ok(c) => return Ok(c),
Err(V2Dial::Failed(e)) => return Err(e),
Err(V2Dial::Unsupported) if self.legacy_client.is_none() => {
return Err(format!(
"{addr} does not speak zk-quic/2 and legacy zk-quic is off"
))
}
Err(V2Dial::Unsupported) => {}
}
}
let Some(legacy) = self.legacy_client.clone() else {
return Err(format!(
"{addr}: peer node key unknown and legacy zk-quic is off"
));
};
let conn = self
.endpoint
.connect_with(legacy, addr, "localhost")
.map_err(|e| format!("connect {}: {}", addr, e))?
.await
.map_err(|e| format!("handshake {}: {}", addr, e))?;
eprintln!(" [QUIC] connected to {addr} ({})", proto_name(&conn));
Ok(conn)
}
async fn dial_v2(
&self,
addr: SocketAddr,
expected_fingerprint: &str,
) -> Result<quinn::Connection, V2Dial> {
let failed = |e: String| V2Dial::Failed(format!("{addr}: {e}"));
let cfg = make_v2_client_config(
self.node_cert.clone(),
self.node_key.pkcs8_bytes(),
expected_fingerprint.to_string(),
)
.map_err(failed)?;
let connecting = self
.endpoint
.connect_with(cfg, addr, "localhost")
.map_err(|e| failed(format!("connect: {e}")))?;
match connecting.await {
Ok(conn) => {
eprintln!(" [QUIC] connected to {addr} ({})", proto_name(&conn));
Ok(conn)
}
Err(quinn::ConnectionError::ConnectionClosed(c))
if u64::from(c.error_code) == NO_APPLICATION_PROTOCOL =>
{
Err(V2Dial::Unsupported)
}
Err(e) => Err(failed(format!("handshake: {e}"))),
}
}
fn note_refused(&self, addr: SocketAddr, conn: &quinn::Connection) -> bool {
if !is_v2_refused(conn) || self.legacy_client.is_none() {
return false;
}
eprintln!(" [QUIC] {addr} refused zk-quic/2 (not rostered there?); using legacy zk-quic");
self.v2_cooldown
.insert(addr, std::time::Instant::now() + V2_COOLDOWN);
self.connections.remove(&addr);
true
}
fn note_v2_refused(&self, addr: SocketAddr) -> bool {
let conn = self.connections.get(&addr).map(|c| c.clone());
conn.is_some_and(|c| self.note_refused(addr, &c))
}
pub fn connection_protocols(&self) -> Vec<(SocketAddr, &'static str)> {
self.connections
.iter()
.map(|e| (*e.key(), proto_name(e.value())))
.collect()
}
pub fn local_addr(&self) -> Option<SocketAddr> {
self.endpoint.local_addr().ok()
}
pub fn broadcast_offer_to_subscribers(
&self,
offer: &TaskOffer,
) -> Option<(String, TaskResult)> {
let subs: Vec<(String, SubscriberOfferTx)> = self
.subscriber_txs
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
if subs.is_empty() {
return None;
}
let (reply_tx, reply_rx) = mpsc::channel();
let mut dead: Vec<String> = Vec::new();
for (url, offer_tx) in &subs {
if offer_tx.send((offer.clone(), reply_tx.clone())).is_err() {
dead.push(url.clone());
}
}
for url in &dead {
self.subscriber_txs.remove(url);
eprintln!(" [QUIC] pruned disconnected subscriber {}", url);
}
drop(reply_tx); let n = subs.len() - dead.len();
for _ in 0..n {
if let Ok((url, Ok(result))) = reply_rx.recv() {
return Some((url, result));
}
}
None
}
pub fn subscriber_urls(&self) -> Vec<String> {
self.subscriber_txs
.iter()
.map(|e| e.key().clone())
.collect()
}
pub fn connect_and_subscribe(
&self,
peer_addr: SocketAddr,
our_url: &str,
) -> Result<SubFeed, String> {
self.runtime
.block_on(self.connect_and_subscribe_async(peer_addr, our_url))
}
async fn connect_and_subscribe_async(
&self,
peer_addr: SocketAddr,
our_url: &str,
) -> Result<SubFeed, String> {
let conn = self.dial(peer_addr).await?;
let peer = match (conn_alpn(&conn).as_deref(), self.state.upgrade()) {
(Some(a), Some(state)) if a == ALPN_V2 => conn_peer_cert(&conn)
.and_then(|c| cert_ed25519_pubkey(c.as_ref()))
.and_then(|pk| rostered_peer(&state, &pk).ok()),
_ => None,
};
let (mut send, recv) = conn
.open_bi()
.await
.map_err(|e| format!("open stream: {}", e))?;
let mut sub = SubscribeMessage {
action: "subscribe".to_string(),
peer_url: our_url.to_string(),
peer_key: self.peer_key.clone(),
node_sig: None,
};
sub.sign_for_quic(&self.node_key);
let msg = serde_json::to_vec(&sub).map_err(|e| e.to_string())?;
write_msg_keep_open(&mut send, &msg).await?;
Ok(SubFeed {
send,
recv,
conn,
peer,
})
}
}
pub fn run_subscription_client(state: Arc<BrokerState>, our_url: String, execute_fn: ExecuteFn) {
let spawned = Arc::new(Mutex::new(HashSet::<String>::new()));
thread::spawn(move || {
let mut interval = 0u32;
loop {
thread::sleep(Duration::from_millis(if interval < 20 {
2000
} else {
10000
}));
interval = interval.saturating_add(1);
let transport = match state.quic.get().cloned() {
Some(t) => t,
None => continue,
};
let peer_urls: Vec<String> = state.peer_manager.peer_urls();
for peer_url in peer_urls {
if peer_url == our_url {
continue;
}
let quic_port = state.peer_manager.get_quic_port(&peer_url);
if quic_port == 0 {
continue;
}
let url_trimmed = peer_url
.trim_start_matches("http://")
.trim_start_matches("https://");
let host = url_trimmed.split(':').next().unwrap_or("127.0.0.1");
let connect_host = if host.starts_with("127.") {
"127.0.0.1"
} else {
host
};
let addr: SocketAddr = match resolve_peer_addr(connect_host, quic_port) {
Some(a) => a,
None => {
eprintln!(
" [QUIC subscribe] {}:{}: could not resolve — skipping",
connect_host, quic_port
);
continue;
}
};
{
let mut g = spawned.lock().unwrap();
if g.contains(&peer_url) {
continue;
}
g.insert(peer_url.clone());
}
let transport_c = transport.clone();
let state_c = state.clone();
let our_url_c = our_url.clone();
let execute_fn_c = execute_fn.clone();
let peer_url_c = peer_url.clone();
let spawned_c = spawned.clone();
thread::spawn(move || {
run_subscription_to_peer(
transport_c,
state_c,
addr,
&peer_url_c,
&our_url_c,
execute_fn_c,
);
spawned_c.lock().unwrap().remove(&peer_url_c);
});
}
}
});
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FeedAttempt {
ConnectFailed,
Closed,
}
fn keep_subscribing(cached_quic_port: u16) -> bool {
cached_quic_port > 0
}
fn run_feed_subscription(
cached_quic_port: impl Fn() -> u16,
mut attempt: impl FnMut() -> FeedAttempt,
pause: impl Fn(Duration),
) {
while keep_subscribing(cached_quic_port()) {
pause(match attempt() {
FeedAttempt::ConnectFailed => Duration::from_secs(5),
FeedAttempt::Closed => Duration::from_secs(2),
});
}
}
fn run_subscription_to_peer(
transport: Arc<QuicTransport>,
state: Arc<BrokerState>,
peer_addr: SocketAddr,
peer_url: &str,
our_url: &str,
execute_fn: ExecuteFn,
) {
let runtime = transport.runtime.handle().clone();
run_feed_subscription(
|| state.peer_manager.get_quic_port(peer_url),
|| {
let SubFeed {
mut send,
mut recv,
conn,
peer,
} = match transport.connect_and_subscribe(peer_addr, our_url) {
Ok(feed) => feed,
Err(e) => {
eprintln!(" [QUIC subscribe] {}: {}", peer_addr, e);
return FeedAttempt::ConnectFailed;
}
};
eprintln!(" [QUIC subscribe] Subscribed to {} (task feed)", peer_addr);
while let Ok(msg) = runtime.block_on(read_msg(&mut recv)) {
let offer: TaskOffer = match serde_json::from_slice(&msg) {
Ok(o) => o,
Err(_) => continue,
};
let base = transport.gate_flags(&state);
let conn_fp = peer.as_ref().map(|p| p.fingerprint.as_str());
let (flags, gated) =
match flags_for_conn(base, peer.as_ref(), offer.node_sig.as_ref()) {
Ok(f) => (f, gate_offer_from(&state, &msg, &offer, f, conn_fp)),
Err(reason) => (
base,
Err(TaskReject {
task_id: offer.task_id.clone(),
reason,
}),
),
};
let resp = match gated.and_then(|requester_fp| {
execute_fn(
&state,
&offer,
flags.require_voucher,
requester_fp.as_deref(),
)
}) {
Ok(r) => serde_json::to_vec(&r).unwrap_or_default(),
Err(rej) => serde_json::to_vec(&rej).unwrap_or_default(),
};
if runtime
.block_on(write_msg_keep_open(&mut send, &resp))
.is_err()
{
break;
}
}
transport.note_refused(peer_addr, &conn);
FeedAttempt::Closed
},
thread::sleep,
);
eprintln!(
" [QUIC subscribe] {}: stopped; {} no longer advertises a QUIC port",
peer_addr, peer_url
);
}
impl Drop for QuicTransport {
fn drop(&mut self) {
self.endpoint.close(0u32.into(), b"shutdown");
}
}
async fn accept_loop(transport: Arc<QuicTransport>, state: Arc<BrokerState>) {
loop {
let incoming = match transport.endpoint.accept().await {
Some(i) => i,
None => break,
};
let transport_clone = transport.clone();
let state_clone = state.clone();
tokio::spawn(async move {
let conn = match incoming.await {
Ok(c) => c,
Err(e) => {
eprintln!(" [QUIC] accept error: {}", e);
return;
}
};
let peer = if conn_alpn(&conn).as_deref() == Some(ALPN_V2) {
match verified_peer(&state_clone, &conn) {
Ok(p) => Some(Arc::new(p)),
Err(why) => {
eprintln!(" [QUIC] refused zk-quic/2 client: {why}");
conn.close(V2_REFUSED.into(), why.as_bytes());
return;
}
}
} else {
None
};
loop {
let stream = match conn.accept_bi().await {
Ok(s) => s,
Err(quinn::ConnectionError::ApplicationClosed(_)) => break,
Err(_) => break,
};
let t = transport_clone.clone();
let st = state_clone.clone();
let peer = peer.clone();
tokio::spawn(async move {
handle_incoming_stream(stream, t, st, peer).await;
});
}
});
}
}
async fn handle_incoming_stream(
(mut send, mut recv): (quinn::SendStream, quinn::RecvStream),
transport: Arc<QuicTransport>,
state: Arc<BrokerState>,
peer: Option<Arc<VerifiedPeer>>,
) {
let msg = match read_msg(&mut recv).await {
Ok(m) => m,
Err(_) => return,
};
let expected_key = state.peer_manager.peer_key();
if let Ok(sub) = serde_json::from_slice::<SubscribeMessage>(&msg) {
if sub.action == "subscribe" && !sub.peer_url.is_empty() {
if !conn_peer_key_ok(expected_key, &sub.peer_key, peer.is_some()) {
eprintln!(
" [QUIC] rejected subscribe from {}: bad peer_key",
sub.peer_url
);
return;
}
let gated = flags_for_conn(
transport.gate_flags(&state),
peer.as_deref(),
sub.node_sig.as_ref(),
)
.and_then(|f| quic_subscribe_gate(&state, &msg, &sub, f));
if let Err(reason) = gated {
eprintln!(
" [QUIC] refused subscribe from {}: {}",
sub.peer_url, reason
);
let _ = send.reset(1u32.into());
let _ = recv.stop(1u32.into());
return;
}
let peer_url = sub.peer_url.clone();
let (offer_tx, offer_rx) = mpsc::channel();
let runtime_handle = transport.runtime.handle().clone();
let node_key = transport.node_key.clone();
thread::spawn(move || {
subscriber_loop(offer_rx, send, recv, peer_url, runtime_handle, node_key);
});
transport.subscriber_txs.insert(sub.peer_url, offer_tx);
return;
}
}
let offer: TaskOffer = match serde_json::from_slice(&msg) {
Ok(o) => o,
Err(e) => {
let r = TaskReject {
task_id: String::new(),
reason: format!("bad payload: {}", e),
};
let _ = write_msg(&mut send, &serde_json::to_vec(&r).unwrap()).await;
return;
}
};
if !conn_peer_key_ok(expected_key, &offer.peer_key, peer.is_some()) {
let r = TaskReject {
task_id: offer.task_id.clone(),
reason: "unauthorized peer".into(),
};
let _ = write_msg(&mut send, &serde_json::to_vec(&r).unwrap()).await;
return;
}
let base = transport.gate_flags(&state);
let flags = match flags_for_conn(base, peer.as_deref(), offer.node_sig.as_ref()) {
Ok(f) => f,
Err(reason) => {
let r = TaskReject {
task_id: offer.task_id.clone(),
reason,
};
let _ = write_msg(&mut send, &serde_json::to_vec(&r).unwrap()).await;
return;
}
};
let conn_fp = peer.as_ref().map(|p| p.fingerprint.clone());
let requester_fp = match gate_offer_from(&state, &msg, &offer, flags, conn_fp.as_deref()) {
Ok(fp) => fp,
Err(r) => {
let _ = write_msg(&mut send, &serde_json::to_vec(&r).unwrap()).await;
return;
}
};
handle_stream(
send,
recv,
offer,
state,
flags.require_voucher,
requester_fp,
)
.await;
}
fn gate_offer_from(
state: &Arc<BrokerState>,
raw: &[u8],
offer: &TaskOffer,
flags: PeerGateFlags,
conn_fp: Option<&str>,
) -> Result<Option<String>, TaskReject> {
quic_offer_gate_from(state, raw, offer, flags, conn_fp).map_err(|reason| TaskReject {
task_id: offer.task_id.clone(),
reason,
})
}
#[cfg(test)]
fn gate_offer(
state: &Arc<BrokerState>,
raw: &[u8],
offer: &TaskOffer,
flags: PeerGateFlags,
) -> Option<TaskReject> {
gate_offer_from(state, raw, offer, flags, None).err()
}
fn subscriber_loop(
offer_rx: OfferReceiver,
mut send: quinn::SendStream,
mut recv: quinn::RecvStream,
peer_url: String,
runtime_handle: tokio::runtime::Handle,
node_key: Arc<NodeKey>,
) {
while let Ok((mut offer, reply_tx)) = offer_rx.recv() {
offer.sign_for_quic(&node_key);
let payload = match serde_json::to_vec(&offer) {
Ok(p) => p,
Err(_) => continue,
};
let write_read = async {
write_msg_keep_open(&mut send, &payload).await?;
read_msg(&mut recv).await
};
let resp_bytes = match runtime_handle.block_on(write_read) {
Ok(b) => b,
Err(_) => break,
};
let result = if let Ok(r) = serde_json::from_slice::<TaskResult>(&resp_bytes) {
Ok(r)
} else if let Ok(rej) = serde_json::from_slice::<TaskReject>(&resp_bytes) {
Err(rej)
} else {
continue;
};
let _ = reply_tx.send((peer_url.clone(), result));
}
}
async fn handle_stream(
mut send: quinn::SendStream,
_recv: quinn::RecvStream,
offer: TaskOffer,
state: Arc<BrokerState>,
require_voucher: bool,
requester_fp: Option<String>,
) {
let task_id = offer.task_id.clone();
let outcome = tokio::task::spawn_blocking(move || {
run_peer_offer_from(&state, &offer, require_voucher, requester_fp.as_deref())
})
.await;
let resp_bytes = match outcome {
Ok(Ok(PeerOfferOutcome::Reserved(accept))) => serde_json::to_vec(&accept),
Ok(Ok(PeerOfferOutcome::Executed(result))) => serde_json::to_vec(&result),
Ok(Err(reject)) => serde_json::to_vec(&reject),
Err(e) => serde_json::to_vec(&TaskReject {
task_id,
reason: format!("internal: {}", e),
}),
};
if let Ok(bytes) = resp_bytes {
let _ = write_msg(&mut send, &bytes).await;
}
}
#[cfg(test)]
mod panic_fix_tests {
use super::{cap_msg_len, peer_key_ok};
use crate::broker::task_board::{SubscribeMessage, TaskOffer};
#[test]
fn cap_msg_len_bounds() {
assert_eq!(cap_msg_len(1024u32.to_be_bytes()).unwrap(), 1024usize);
assert_eq!(cap_msg_len(0u32.to_be_bytes()).unwrap(), 0usize);
let over = (super::MAX_QUIC_MSG_BYTES as u32) + 1;
assert!(cap_msg_len(over.to_be_bytes()).is_err());
}
#[test]
fn a_keyless_broker_accepts_no_peer() {
assert!(!peer_key_ok("", ""));
assert!(!peer_key_ok("", "whatever"));
}
#[test]
fn matching_key_is_accepted() {
assert!(peer_key_ok("s3cret", "s3cret"));
}
#[test]
fn mismatched_or_missing_key_is_rejected() {
assert!(!peer_key_ok("s3cret", "wrong"));
assert!(!peer_key_ok("s3cret", ""));
}
#[test]
fn task_offer_peer_key_defaults_when_absent() {
let json = r#"{
"task_id":"t1","payload_b64":"","max_price_per_hour":1.0,
"estimated_duration_secs":1.0,"timeout_secs":10.0,
"requester_user_id":"u","source_broker":"n"
}"#;
let offer: TaskOffer = serde_json::from_str(json).unwrap();
assert_eq!(offer.peer_key, "");
}
#[test]
fn task_offer_peer_key_round_trips() {
let json = r#"{
"task_id":"t1","payload_b64":"","max_price_per_hour":1.0,
"estimated_duration_secs":1.0,"timeout_secs":10.0,
"requester_user_id":"u","source_broker":"n","peer_key":"abc"
}"#;
let offer: TaskOffer = serde_json::from_str(json).unwrap();
assert_eq!(offer.peer_key, "abc");
let back = serde_json::to_string(&offer).unwrap();
assert!(back.contains("\"peer_key\":\"abc\""));
}
#[test]
fn subscribe_message_peer_key_defaults_and_round_trips() {
let absent: SubscribeMessage =
serde_json::from_str(r#"{"action":"subscribe","peer_url":"http://x"}"#).unwrap();
assert_eq!(absent.peer_key, "");
let present: SubscribeMessage =
serde_json::from_str(r#"{"action":"subscribe","peer_url":"http://x","peer_key":"k"}"#)
.unwrap();
assert_eq!(present.peer_key, "k");
assert!(present.node_sig.is_none());
assert!(!serde_json::to_string(&present)
.unwrap()
.contains("node_sig"));
}
#[test]
fn dead_subscribers_are_pruned_on_send_failure() {
use dashmap::DashMap;
use std::sync::mpsc;
let subs: DashMap<String, mpsc::Sender<u8>> = DashMap::new();
let (live_tx, _live_rx) = mpsc::channel::<u8>();
let (dead_tx, dead_rx) = mpsc::channel::<u8>();
subs.insert("live".into(), live_tx);
subs.insert("dead".into(), dead_tx);
drop(dead_rx);
let snapshot: Vec<(String, mpsc::Sender<u8>)> = subs
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
let mut dead = Vec::new();
for (url, tx) in &snapshot {
if tx.send(1).is_err() {
dead.push(url.clone());
}
}
for url in &dead {
subs.remove(url);
}
assert!(subs.contains_key("live"));
assert!(!subs.contains_key("dead"), "dead subscriber must be pruned");
}
#[test]
fn quic_seed_is_stable_and_key_separated() {
assert_eq!(super::quic_seed("x"), super::quic_seed("x"));
assert_ne!(super::quic_seed("x"), super::quic_seed("y"));
}
#[test]
fn derive_quic_identity_is_deterministic() {
let (c1, _k1) = super::derive_quic_identity("shared-secret").expect("derive");
let (c2, _k2) = super::derive_quic_identity("shared-secret").expect("derive");
assert_eq!(
c1.as_ref(),
c2.as_ref(),
"same peer_key must yield byte-identical cert"
);
assert_eq!(super::cert_sha256(&c1), super::cert_sha256(&c2));
}
#[test]
fn derive_quic_identity_differs_by_key() {
let (a, _) = super::derive_quic_identity("key-a").expect("derive a");
let (b, _) = super::derive_quic_identity("key-b").expect("derive b");
assert_ne!(
super::cert_sha256(&a),
super::cert_sha256(&b),
"different peer_key must yield a different cert"
);
}
#[test]
fn derived_key_is_a_valid_private_key() {
let (_c, k) = super::derive_quic_identity("shared-secret").expect("derive");
assert!(matches!(k, rustls::pki_types::PrivateKeyDer::Pkcs8(_)));
}
#[test]
fn pinned_verify_accepts_matching_cert() {
use rustls::client::danger::ServerCertVerifier;
use rustls::pki_types::{ServerName, UnixTime};
let (cert, _k) = super::derive_quic_identity("shared-secret").expect("derive");
let v = super::PinnedVerify {
expected: super::cert_sha256(&cert),
};
let name = ServerName::try_from("localhost").unwrap();
let r = v.verify_server_cert(&cert, &[], &name, &[], UnixTime::now());
assert!(r.is_ok(), "matching cert must verify");
}
#[test]
fn pinned_verify_rejects_mismatched_cert() {
use rustls::client::danger::ServerCertVerifier;
use rustls::pki_types::{ServerName, UnixTime};
let (good, _) = super::derive_quic_identity("shared-secret").expect("derive");
let (attacker, _) = super::derive_quic_identity("attacker-key").expect("derive");
let v = super::PinnedVerify {
expected: super::cert_sha256(&good),
};
let name = ServerName::try_from("localhost").unwrap();
let r = v.verify_server_cert(&attacker, &[], &name, &[], UnixTime::now());
assert!(r.is_err(), "a cert not matching the pin must be rejected");
}
#[test]
fn legacy_identity_needs_a_peer_key() {
assert!(super::build_legacy("shared-secret", super::derive_quic_identity).is_ok());
let err = super::build_legacy("", super::derive_quic_identity)
.map(|_| ())
.expect_err("keyless must not build");
assert!(err.contains("ZAKURO_PEER_KEY"), "{err}");
}
#[test]
fn a_failed_cert_derivation_builds_no_legacy_identity() {
let err = super::build_legacy("k", |_| Err("boom".into()))
.map(|_| ())
.expect_err("derivation failure must not build");
assert!(err.contains("boom"), "{err}");
}
#[test]
fn a_server_with_no_protocol_has_no_tls_config() {
let err = super::make_server_config(None, None)
.map(|_| ())
.expect_err("must refuse");
assert!(err.contains("no QUIC protocol"), "{err}");
}
#[test]
fn legacy_switch_parses_off_values_only() {
assert!(super::legacy_allowed(None));
assert!(super::legacy_allowed(Some("1")));
assert!(super::legacy_allowed(Some("")));
assert!(!super::legacy_allowed(Some("0")));
assert!(!super::legacy_allowed(Some("false")));
}
#[test]
fn a_node_cert_carries_the_node_key_and_its_fingerprint() {
let key = crate::broker::node_identity::NodeKey::generate();
let cert = super::node_cert(&key).unwrap();
let pk = super::cert_ed25519_pubkey(cert.as_ref()).expect("Ed25519 key in cert");
assert_eq!(super::pubkey_fingerprint(&pk), key.fingerprint());
assert!(super::cert_ed25519_pubkey(b"not a cert").is_none());
}
#[test]
fn a_verified_connection_stands_in_for_the_node_signature() {
use crate::broker::server::PeerGateFlags;
use crate::broker::task_board::OfferNodeSig;
let base = PeerGateFlags {
enforce_node_sig: true,
require_voucher: true,
};
let peer = super::VerifiedPeer {
pubkey_b64: "pk".into(),
fingerprint: "fp".into(),
};
let sig = |id: &str| OfferNodeSig {
node_id: id.into(),
sig: String::new(),
nonce: String::new(),
ts: 0,
};
assert_eq!(super::flags_for_conn(base, None, None), Ok(base));
let f = super::flags_for_conn(base, Some(&peer), None).unwrap();
assert!(!f.enforce_node_sig && f.require_voucher);
let lax = PeerGateFlags {
enforce_node_sig: false,
..base
};
let f = super::flags_for_conn(lax, Some(&peer), Some(&sig("pk"))).unwrap();
assert!(f.enforce_node_sig);
assert!(super::flags_for_conn(base, Some(&peer), Some(&sig("other"))).is_err());
}
#[test]
fn a_keyless_broker_takes_a_verified_node_but_nobody_else() {
assert!(super::conn_peer_key_ok("", "", true));
assert!(!super::conn_peer_key_ok("", "", false));
assert!(!super::conn_peer_key_ok("k", "", true));
assert!(super::conn_peer_key_ok("k", "k", true));
}
#[test]
fn task_feed_config_keeps_idle_connections_alive() {
let rendered = format!("{:?}", super::task_feed_transport_config());
assert!(
rendered.contains("keep_alive_interval: Some("),
"task feed needs a keepalive or idle subscriptions die: {rendered}"
);
assert!(
rendered.contains("max_idle_timeout: Some("),
"task feed needs an explicit idle timeout so it does not rely on a \
default that may be shorter than the keepalive: {rendered}"
);
}
#[test]
fn resolve_peer_addr_accepts_literal_ips() {
let addr = super::resolve_peer_addr("10.13.13.15", 9001)
.expect("a literal IP must resolve without a DNS lookup");
assert_eq!(addr.to_string(), "10.13.13.15:9001");
}
#[test]
fn resolve_peer_addr_accepts_dns_names() {
let addr = super::resolve_peer_addr("localhost", 9001)
.expect("a DNS name must resolve — parse::<SocketAddr>() cannot do this");
assert_eq!(addr.port(), 9001);
assert!(
addr.ip().is_loopback(),
"localhost should be loopback: {addr}"
);
}
#[test]
fn resolve_peer_addr_prefers_ipv4() {
let addr = super::resolve_peer_addr("localhost", 9001).expect("localhost resolves");
assert!(addr.is_ipv4(), "must prefer the v4 address, got {addr}");
}
#[test]
fn resolve_peer_addr_rejects_unresolvable_names() {
assert!(super::resolve_peer_addr("no-such-peer.invalid", 9001).is_none());
}
}
#[cfg(test)]
mod feed_subscription_tests {
use super::{keep_subscribing, run_feed_subscription, FeedAttempt};
use crate::broker::peer::PeerManager;
use std::cell::{Cell, RefCell};
use std::time::Duration;
const PEER: &str = "http://10.13.13.7:9000";
#[test]
fn a_feed_subscription_keeps_going_only_while_the_peer_advertises_a_quic_port() {
assert!(keep_subscribing(9001));
assert!(!keep_subscribing(0));
}
#[test]
fn the_subscription_loop_ends_once_the_peers_cached_port_reads_0() {
let pm = PeerManager::new(Some("10.13.13.5"), &[], 9000, "k".into(), true);
pm.set_quic_port(PEER, 9001);
let attempts = Cell::new(0u32);
let pauses = RefCell::new(Vec::new());
run_feed_subscription(
|| pm.get_quic_port(PEER),
|| {
attempts.set(attempts.get() + 1);
assert!(attempts.get() < 10, "the loop ignored the cleared port");
if attempts.get() == 1 {
return FeedAttempt::Closed;
}
pm.set_quic_port(PEER, 0);
FeedAttempt::ConnectFailed
},
|d| pauses.borrow_mut().push(d),
);
assert_eq!(attempts.get(), 2);
assert_eq!(
*pauses.borrow(),
vec![Duration::from_secs(2), Duration::from_secs(5)]
);
}
#[test]
fn the_subscription_loop_makes_no_attempt_without_a_cached_port() {
let pm = PeerManager::new(Some("10.13.13.5"), &[], 9000, "k".into(), true);
run_feed_subscription(
|| pm.get_quic_port(PEER),
|| panic!("dialed the task feed of a peer with no QUIC port"),
|_| {},
);
}
}
#[cfg(test)]
mod offer_gate_tests {
use super::{gate_offer, gate_offer_from, quic_subscribe_gate, QuicTransport};
use crate::broker::node_identity::NodeKey;
use crate::broker::server::PeerGateFlags;
use crate::broker::task_board::{SubscribeMessage, TaskOffer};
use crate::broker::{BrokerConfig, BrokerState};
use std::sync::Arc;
const ARMED: PeerGateFlags = PeerGateFlags {
enforce_node_sig: true,
require_voucher: true,
};
const SIG_ONLY: PeerGateFlags = PeerGateFlags {
enforce_node_sig: true,
require_voucher: false,
};
const VOUCHER_ONLY: PeerGateFlags = PeerGateFlags {
enforce_node_sig: false,
require_voucher: true,
};
const UNSET: PeerGateFlags = PeerGateFlags {
enforce_node_sig: false,
require_voucher: false,
};
const MESH_KEY: &str = "test-mesh-key";
fn keyed_state() -> Arc<BrokerState> {
Arc::new(BrokerState::with_config(BrokerConfig {
peer_key: Some(MESH_KEY.into()),
..Default::default()
}))
}
const OLD_OFFER: &str = r#"{"task_id":"t1","payload_b64":"","max_price_per_hour":100.0,"estimated_duration_secs":1.0,"timeout_secs":10.0,"requester_user_id":"u","source_broker":"n","peer_key":"k","currency":"jpy"}"#;
fn receiver(sender: &NodeKey) -> (Arc<BrokerState>, NodeKey) {
let st = BrokerState::with_config(BrokerConfig {
peer_key: Some(MESH_KEY.into()),
api_url: Some("http://127.0.0.1:1".into()),
..Default::default()
});
st.node_roster.insert(sender.public_b64(), false);
let hub = NodeKey::generate();
*st.dash_voucher_pubkey.write().unwrap() = vec![hub.public_b64()];
(Arc::new(st), hub)
}
fn old_offer() -> TaskOffer {
serde_json::from_str(OLD_OFFER).unwrap()
}
fn vouched_offer(hub: &NodeKey) -> TaskOffer {
let signed =
r#"{"v":2,"requester_user":"u","budget":1000000.0,"task_nonce":"n","exp":9999999999}"#;
TaskOffer {
voucher_signed_json: signed.to_string(),
voucher_sig: hub.sign(signed.as_bytes()),
..old_offer()
}
}
fn send(offer: &TaskOffer, key: &NodeKey) -> Vec<u8> {
let mut o = offer.clone();
o.sign_for_quic(key);
serde_json::to_vec(&o).unwrap()
}
fn receive(st: &Arc<BrokerState>, raw: &[u8], flags: PeerGateFlags) -> Result<(), String> {
let offer: TaskOffer = serde_json::from_slice(raw).unwrap();
match gate_offer(st, raw, &offer, flags) {
None => Ok(()),
Some(rej) => {
assert_eq!(rej.task_id, "t1");
Err(rej.reason)
}
}
}
#[test]
fn enforced_node_sig_refuses_an_unsigned_offer() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let err = receive(&st, OLD_OFFER.as_bytes(), SIG_ONLY).unwrap_err();
assert!(err.contains("node signature check failed"), "{err}");
}
#[test]
fn enforced_node_sig_refuses_a_tampered_offer() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let raw = send(&old_offer(), &sender);
let mut v: serde_json::Value = serde_json::from_slice(&raw).unwrap();
v["max_price_per_hour"] = serde_json::json!(1.0e9);
let tampered = serde_json::to_vec(&v).unwrap();
let err = receive(&st, &tampered, SIG_ONLY).unwrap_err();
assert!(err.contains("bad signature"), "{err}");
}
#[test]
fn enforced_node_sig_refuses_an_unrostered_signer() {
let (st, _) = receiver(&NodeKey::generate());
let raw = send(&old_offer(), &NodeKey::generate());
let err = receive(&st, &raw, SIG_ONLY).unwrap_err();
assert!(err.contains("not in roster"), "{err}");
}
#[test]
fn enforced_node_sig_refuses_a_replayed_offer() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let raw = send(&old_offer(), &sender);
assert_eq!(receive(&st, &raw, SIG_ONLY), Ok(()));
let err = receive(&st, &raw, SIG_ONLY).unwrap_err();
assert!(err.contains("replayed"), "{err}");
}
#[test]
fn an_http_signature_does_not_pass_as_a_quic_one() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let mut offer = old_offer();
offer.sign_for_quic(&sender);
let raw = serde_json::to_vec(&offer).unwrap();
let body = crate::broker::task_board::quic_offer_sign_bytes(&raw).unwrap();
let hdrs = sender.sign_headers("POST", "/peer/tasks/offer", &body);
let sig = hdrs.iter().find(|(k, _)| k == "X-Node-Sig").unwrap();
offer.node_sig.as_mut().unwrap().sig = sig.1.clone();
let raw = serde_json::to_vec(&offer).unwrap();
let err = receive(&st, &raw, SIG_ONLY).unwrap_err();
assert!(err.contains("bad signature"), "{err}");
}
fn bare_subscribe() -> SubscribeMessage {
SubscribeMessage {
action: "subscribe".into(),
peer_url: "http://10.0.0.2:8080".into(),
peer_key: MESH_KEY.into(),
node_sig: None,
}
}
fn subscribe_wire(key: &NodeKey) -> Vec<u8> {
let mut s = bare_subscribe();
s.sign_for_quic(key);
serde_json::to_vec(&s).unwrap()
}
fn admit(st: &Arc<BrokerState>, raw: &[u8], flags: PeerGateFlags) -> Result<(), String> {
let sub: SubscribeMessage = serde_json::from_slice(raw).unwrap();
quic_subscribe_gate(st, raw, &sub, flags)
}
#[test]
fn enforced_node_sig_refuses_an_unsigned_subscribe() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let raw = serde_json::to_vec(&bare_subscribe()).unwrap();
let err = admit(&st, &raw, SIG_ONLY).unwrap_err();
assert!(err.contains("node signature check failed"), "{err}");
}
#[test]
fn enforced_node_sig_refuses_a_tampered_subscribe() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let mut v: serde_json::Value = serde_json::from_slice(&subscribe_wire(&sender)).unwrap();
v["peer_url"] = serde_json::json!("http://attacker:1");
let err = admit(&st, &serde_json::to_vec(&v).unwrap(), SIG_ONLY).unwrap_err();
assert!(err.contains("bad signature"), "{err}");
}
#[test]
fn enforced_node_sig_refuses_an_unrostered_subscriber() {
let (st, _) = receiver(&NodeKey::generate());
let err = admit(&st, &subscribe_wire(&NodeKey::generate()), SIG_ONLY).unwrap_err();
assert!(err.contains("not in roster"), "{err}");
}
#[test]
fn enforced_node_sig_refuses_a_replayed_subscribe() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let raw = subscribe_wire(&sender);
assert_eq!(admit(&st, &raw, SIG_ONLY), Ok(()));
let err = admit(&st, &raw, SIG_ONLY).unwrap_err();
assert!(err.contains("replayed"), "{err}");
}
#[test]
fn an_http_or_offer_signature_does_not_pass_for_the_subscribe_path() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
for path in ["/peer/tasks/offer", "/quic/tasks/offer"] {
let mut sub = bare_subscribe();
sub.sign_for_quic(&sender);
let raw = serde_json::to_vec(&sub).unwrap();
let body = crate::broker::task_board::quic_offer_sign_bytes(&raw).unwrap();
let hdrs = sender.sign_headers("POST", path, &body);
sub.node_sig.as_mut().unwrap().sig = hdrs
.iter()
.find(|(k, _)| k == "X-Node-Sig")
.unwrap()
.1
.clone();
sub.node_sig.as_mut().unwrap().nonce = hdrs
.iter()
.find(|(k, _)| k == "X-Node-Nonce")
.unwrap()
.1
.clone();
sub.node_sig.as_mut().unwrap().ts = hdrs
.iter()
.find(|(k, _)| k == "X-Node-Ts")
.unwrap()
.1
.parse()
.unwrap();
let err = admit(&st, &serde_json::to_vec(&sub).unwrap(), SIG_ONLY).unwrap_err();
assert!(err.contains("bad signature"), "{path}: {err}");
}
}
#[test]
fn a_signed_rostered_subscribe_passes_and_an_unset_gate_admits_an_old_one() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
assert_eq!(admit(&st, &subscribe_wire(&sender), SIG_ONLY), Ok(()));
let old = serde_json::to_vec(&bare_subscribe()).unwrap();
assert_eq!(admit(&st, &old, UNSET), Ok(()));
}
fn wait_subscribers(q: &QuicTransport, n: usize) -> usize {
for _ in 0..50 {
if q.subscriber_urls().len() >= n {
break;
}
std::thread::sleep(std::time::Duration::from_millis(100));
}
q.subscriber_urls().len()
}
#[test]
fn an_armed_publisher_feeds_only_rostered_subscribers_on_the_wire() {
let sub_state = keyed_state();
let (publisher, _) = receiver(&sub_state.node_key);
let lo = "127.0.0.1:0".parse().unwrap();
let qs = QuicTransport::new(lo, sub_state.clone()).unwrap();
let qp = QuicTransport::new(lo, publisher.clone()).unwrap();
qp.set_gate_flags(SIG_ONLY);
let addr = qp.local_addr().unwrap();
let _feed = qs
.connect_and_subscribe(addr, "http://10.0.0.2:8080")
.unwrap();
assert_eq!(wait_subscribers(&qp, 1), 1, "rostered subscriber admitted");
publisher.node_roster.clear();
qp.subscriber_txs.clear();
let _feed = qs
.connect_and_subscribe(addr, "http://10.0.0.3:8080")
.unwrap();
std::thread::sleep(std::time::Duration::from_millis(1500));
assert!(
qp.subscriber_urls().is_empty(),
"unrostered subscriber admitted"
);
assert!(qp.broadcast_offer_to_subscribers(&old_offer()).is_none());
}
#[test]
fn an_offer_without_the_currency_is_refused() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let pre_yen = TaskOffer {
currency: String::new(),
..old_offer()
};
let err = receive(&st, &send(&pre_yen, &sender), UNSET).unwrap_err();
assert!(err.contains("currency mismatch"), "{err}");
}
#[test]
fn required_voucher_refuses_a_foreign_offer_without_one() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let raw = send(&old_offer(), &sender);
let err = receive(&st, &raw, VOUCHER_ONLY).unwrap_err();
assert!(err.contains("voucher check failed"), "{err}");
}
#[test]
fn unset_flags_accept_old_and_badly_signed_offers() {
let (st, _) = receiver(&NodeKey::generate());
assert_eq!(receive(&st, OLD_OFFER.as_bytes(), UNSET), Ok(()));
let raw = send(&old_offer(), &NodeKey::generate());
assert_eq!(receive(&st, &raw, UNSET), Ok(()));
}
#[test]
fn an_upgraded_sender_passes_an_armed_receiver() {
let sender = NodeKey::generate();
let (st, hub) = receiver(&sender);
let raw = send(&vouched_offer(&hub), &sender);
assert_eq!(receive(&st, &raw, ARMED), Ok(()));
}
#[test]
fn the_gate_names_the_requester_it_authenticated() {
let sender = NodeKey::generate();
let (st, hub) = receiver(&sender);
let raw = send(&vouched_offer(&hub), &sender);
let offer: TaskOffer = serde_json::from_slice(&raw).unwrap();
assert_eq!(
gate_offer_from(&st, &raw, &offer, ARMED, Some("1111111111111111")).unwrap(),
Some(sender.fingerprint())
);
let unsigned: TaskOffer = serde_json::from_str(OLD_OFFER).unwrap();
let raw = serde_json::to_vec(&unsigned).unwrap();
let conn = sender.fingerprint();
assert_eq!(
gate_offer_from(&st, &raw, &unsigned, UNSET, Some(&conn)).unwrap(),
Some(conn)
);
assert_eq!(
gate_offer_from(&st, &raw, &unsigned, UNSET, None).unwrap(),
None
);
}
#[test]
fn a_voucher_bound_to_another_node_is_refused_over_quic() {
let sender = NodeKey::generate();
let (st, hub) = receiver(&sender);
let elsewhere = NodeKey::generate().fingerprint();
let signed_json = format!(
r#"{{"v":2,"requester_user":"u","budget":1000000.0,"task_nonce":"n","exp":9999999999,"requester_node":"{elsewhere}"}}"#
);
let mut o = old_offer();
o.voucher_sig = hub.sign(signed_json.as_bytes());
o.voucher_signed_json = signed_json;
let raw = send(&o, &sender);
let offer: TaskOffer = serde_json::from_slice(&raw).unwrap();
let rej = gate_offer_from(&st, &raw, &offer, ARMED, None).unwrap_err();
assert!(
rej.reason.contains("different requester node"),
"{}",
rej.reason
);
let raw = serde_json::to_vec(&o).unwrap();
let rej =
gate_offer_from(&st, &raw, &o, VOUCHER_ONLY, Some(&sender.fingerprint())).unwrap_err();
assert!(
rej.reason.contains("different requester node"),
"{}",
rej.reason
);
let ok = gate_offer_from(&st, &raw, &o, VOUCHER_ONLY, Some(&elsewhere));
assert_eq!(ok.unwrap(), Some(elsewhere));
}
#[test]
fn the_signature_covers_unknown_fields_and_ignores_key_order() {
let sender = NodeKey::generate();
let (st, _) = receiver(&sender);
let mut v: serde_json::Value = serde_json::from_slice(OLD_OFFER.as_bytes()).unwrap();
v["from_a_newer_build"] = serde_json::json!({"b": 2, "a": 1});
let body =
crate::broker::task_board::quic_offer_sign_bytes(&serde_json::to_vec(&v).unwrap())
.unwrap();
let hdrs = sender.sign_headers(
"POST",
crate::broker::task_board::QUIC_OFFER_SIGN_PATH,
&body,
);
let h = |n: &str| hdrs.iter().find(|(k, _)| k == n).unwrap().1.clone();
v["node_sig"] = serde_json::json!({
"node_id": h("X-Node-Id"), "sig": h("X-Node-Sig"),
"nonce": h("X-Node-Nonce"), "ts": h("X-Node-Ts").parse::<u64>().unwrap(),
});
let reordered = format!(
r#"{{"from_a_newer_build":{{"a":1,"b":2}},"node_sig":{},{}"#,
v["node_sig"],
&OLD_OFFER[1..]
);
assert_eq!(receive(&st, reordered.as_bytes(), SIG_ONLY), Ok(()));
let changed = reordered.replace(r#""a":1"#, r#""a":9"#);
assert!(receive(&st, changed.as_bytes(), SIG_ONLY).is_err());
}
#[test]
fn quic_offers_from_an_upgraded_sender_pass_an_armed_receiver_on_the_wire() {
let a = keyed_state();
let (b, hub) = receiver(&a.node_key);
let lo = "127.0.0.1:0".parse().unwrap();
let qa = QuicTransport::new(lo, a.clone()).unwrap();
let qb = QuicTransport::new(lo, b.clone()).unwrap();
qb.set_gate_flags(ARMED);
let addr_b = qb.local_addr().unwrap();
let err = qa.offer_task(addr_b, &vouched_offer(&hub)).unwrap_err();
assert!(err.contains("No matching local worker"), "{err}");
b.node_roster.clear();
let err = qa.offer_task(addr_b, &vouched_offer(&hub)).unwrap_err();
assert!(err.contains("node signature check failed"), "{err}");
}
use crate::broker::task_board::{TaskAccept, TaskReject, TaskResult};
use std::sync::atomic::{AtomicUsize, Ordering};
fn counting_server(count_path: &'static str, body: &'static str) -> (Arc<AtomicUsize>, String) {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
let hits = Arc::new(AtomicUsize::new(0));
let h = hits.clone();
std::thread::spawn(move || {
for req in server.incoming_requests() {
if req.url().contains(count_path) {
h.fetch_add(1, Ordering::SeqCst);
}
let _ = req.respond(tiny_http::Response::from_string(body));
}
});
(hits, format!("http://127.0.0.1:{port}"))
}
fn working_receiver(
sender: &NodeKey,
) -> (
Arc<BrokerState>,
NodeKey,
Arc<AtomicUsize>,
Arc<AtomicUsize>,
) {
let (jobs, worker_url) = counting_server("/execute", "ok");
let (redeems, hub_url) = counting_server("/voucher/redeem", r#"{"redeemed":true}"#);
let st = BrokerState::with_config(BrokerConfig {
peer_key: Some(MESH_KEY.into()),
api_url: Some(hub_url),
api_key: Some("zk_test".into()),
..Default::default()
});
st.set_local_mode(true);
st.node_roster.insert(sender.public_b64(), false);
let hub = NodeKey::generate();
*st.dash_voucher_pubkey.write().unwrap() = vec![hub.public_b64()];
st.workers
.register(crate::broker::worker::test_registration(
"w0", worker_url, 1.0,
));
(Arc::new(st), hub, jobs, redeems)
}
fn wire(
flags: PeerGateFlags,
) -> (
Arc<QuicTransport>,
std::net::SocketAddr,
Arc<BrokerState>,
NodeKey,
Arc<AtomicUsize>,
Arc<AtomicUsize>,
) {
let a = keyed_state();
let (b, hub, jobs, redeems) = working_receiver(&a.node_key);
let lo = "127.0.0.1:0".parse().unwrap();
let qa = QuicTransport::new(lo, a).unwrap();
let qb = QuicTransport::new(lo, b.clone()).unwrap();
qb.set_gate_flags(flags);
let addr = qb.local_addr().unwrap();
std::mem::forget(qb);
(qa, addr, b, hub, jobs, redeems)
}
fn send_raw(qa: &QuicTransport, addr: std::net::SocketAddr, offer: &TaskOffer) -> Vec<u8> {
qa.runtime.block_on(async {
let conn = qa.get_or_connect(addr).await.unwrap();
let (mut s, mut r) = conn.open_bi().await.unwrap();
let mut o = offer.clone();
o.peer_key = qa.peer_key.clone();
o.sign_for_quic(&qa.node_key);
super::write_msg(&mut s, &serde_json::to_vec(&o).unwrap())
.await
.unwrap();
super::read_msg(&mut r).await.unwrap()
})
}
#[test]
fn a_vouched_one_shot_quic_offer_redeems_once_and_a_replay_is_refused() {
let (qa, addr, _b, hub, jobs, redeems) = wire(ARMED);
let offer = vouched_offer(&hub);
let res = qa.offer_task(addr, &offer).unwrap();
assert_eq!(res.task_id, "t1");
assert!(
res.worker_uri.starts_with("zc://worker-"),
"{}",
res.worker_uri
);
assert_eq!(jobs.load(Ordering::SeqCst), 1);
assert_eq!(redeems.load(Ordering::SeqCst), 1, "one run, one redeem");
let err = qa.offer_task(addr, &offer).unwrap_err();
assert!(err.contains("voucher already used"), "{err}");
assert_eq!(jobs.load(Ordering::SeqCst), 1, "a replay must not run");
assert_eq!(
redeems.load(Ordering::SeqCst),
1,
"a replay must not redeem"
);
}
#[test]
fn a_failed_run_releases_the_voucher_for_a_retry() {
let (qa, addr, b, hub, _jobs, redeems) = wire(ARMED);
let mut bad = vouched_offer(&hub);
bad.payload_b64 = "%%% not base64".into();
assert!(qa.offer_task(addr, &bad).is_err());
assert!(b.claimed_voucher_nonces.is_empty());
assert_eq!(redeems.load(Ordering::SeqCst), 0);
qa.offer_task(addr, &vouched_offer(&hub)).unwrap();
assert_eq!(redeems.load(Ordering::SeqCst), 1);
}
#[test]
fn a_quic_offer_pinned_to_another_node_is_refused() {
let (qa, addr, b, _hub, jobs, _) = wire(UNSET);
let mut offer = old_offer();
offer.target_node = "zc://node-0000000000000000".into();
let err = qa.offer_task(addr, &offer).unwrap_err();
assert!(err.contains("pinned to node"), "{err}");
assert_eq!(jobs.load(Ordering::SeqCst), 0);
offer.target_node = b.node_key.node_uri();
qa.offer_task(addr, &offer).unwrap();
assert_eq!(jobs.load(Ordering::SeqCst), 1);
}
#[test]
fn a_two_phase_quic_offer_reserves_like_http() {
let (qa, addr, b, hub, jobs, redeems) = wire(ARMED);
let offer = TaskOffer {
two_phase: true,
..vouched_offer(&hub)
};
let reply = send_raw(&qa, addr, &offer);
let accept: TaskAccept = serde_json::from_slice(&reply).unwrap();
assert_eq!(accept.task_id, "t1");
assert!(serde_json::from_slice::<TaskResult>(&reply).is_err());
assert!(b.pending_offers.contains_key("t1"));
assert_eq!(jobs.load(Ordering::SeqCst), 0, "a reserve must not run");
let reply = send_raw(
&qa,
addr,
&TaskOffer {
task_id: "t2".into(),
..offer.clone()
},
);
let rej: TaskReject = serde_json::from_slice(&reply).unwrap();
assert!(
rej.reason.contains("voucher already used"),
"{}",
rej.reason
);
let pending = crate::broker::server::take_committed(&b, "t1").unwrap();
assert_eq!(pending.offer.task_id, "t1");
assert_eq!(redeems.load(Ordering::SeqCst), 0);
}
#[test]
fn cancelling_a_two_phase_quic_reservation_releases_its_voucher() {
let (qa, addr, b, hub, jobs, _) = wire(ARMED);
let offer = TaskOffer {
two_phase: true,
..vouched_offer(&hub)
};
let _: TaskAccept = serde_json::from_slice(&send_raw(&qa, addr, &offer)).unwrap();
crate::broker::server::cancel_offer(&b, "t1");
assert!(b.claimed_voucher_nonces.is_empty());
qa.offer_task(addr, &vouched_offer(&hub)).unwrap();
assert_eq!(jobs.load(Ordering::SeqCst), 1);
}
use std::net::SocketAddr;
fn lo() -> SocketAddr {
"127.0.0.1:0".parse().unwrap()
}
fn v2_pair(
legacy_a: bool,
legacy_b: bool,
flags: PeerGateFlags,
) -> (
Arc<QuicTransport>,
Arc<QuicTransport>,
SocketAddr,
Arc<BrokerState>,
) {
let a = keyed_state();
let (b, _hub) = receiver(&a.node_key);
let qa = QuicTransport::start(lo(), a, legacy_a).unwrap();
let qb = QuicTransport::start(lo(), b.clone(), legacy_b).unwrap();
qb.set_gate_flags(flags);
let addr = qb.local_addr().unwrap();
qa.fingerprint_hints.insert(addr, b.node_key.fingerprint());
(qa, qb, addr, b)
}
fn protos(q: &QuicTransport) -> Vec<&'static str> {
q.connection_protocols().into_iter().map(|p| p.1).collect()
}
const PAST_GATES: &str = "No matching local worker";
#[test]
fn two_node_keys_complete_a_v2_handshake_and_exchange_offers() {
let (qa, _qb, addr, _b) = v2_pair(true, true, ARMED);
let err = qa.offer_task(addr, &old_offer()).unwrap_err();
assert!(!err.contains("handshake"), "{err}");
assert_eq!(protos(&qa), vec!["zk-quic/2"]);
}
#[test]
fn the_server_learns_the_clients_fingerprint_from_its_certificate() {
let (qa, _qb, addr, b) = v2_pair(true, true, UNSET);
qa.runtime.block_on(async {
let conn = qa.get_or_connect(addr).await.unwrap();
let cert = super::conn_peer_cert(&conn).expect("server cert");
let pk = super::cert_ed25519_pubkey(cert.as_ref()).unwrap();
assert_eq!(super::pubkey_fingerprint(&pk), b.node_key.fingerprint());
});
let client_pk = super::cert_ed25519_pubkey(qa.node_cert.as_ref()).unwrap();
let vp = super::rostered_peer(&b, &client_pk).unwrap();
assert_eq!(vp.fingerprint, qa.node_key.fingerprint());
assert_eq!(vp.pubkey_b64, qa.node_key.public_b64());
}
#[test]
fn a_roster_check_refuses_unknown_and_revoked_keys() {
let k = NodeKey::generate();
let (st, _) = receiver(&NodeKey::generate());
let pk = super::cert_ed25519_pubkey(super::node_cert(&k).unwrap().as_ref()).unwrap();
assert!(super::rostered_peer(&st, &pk)
.unwrap_err()
.contains("not in the roster"));
st.node_roster.insert(k.public_b64(), true);
assert!(super::rostered_peer(&st, &pk)
.unwrap_err()
.contains("revoked"));
st.node_roster.insert(k.public_b64(), false);
assert!(super::rostered_peer(&st, &pk).is_ok());
}
#[test]
fn an_unrostered_v2_client_is_refused() {
let (qa, _qb, addr, b) = v2_pair(false, true, UNSET);
b.node_roster.clear();
assert!(qa.offer_task(addr, &old_offer()).is_err());
assert!(qa.offer_task(addr, &old_offer()).is_err());
assert!(qa.legacy_client.is_none());
}
#[test]
fn a_refused_v2_client_falls_back_to_legacy_when_allowed() {
let (qa, _qb, addr, b) = v2_pair(true, true, UNSET);
b.node_roster.clear();
let err = qa.offer_task(addr, &old_offer()).unwrap_err();
assert!(err.contains(PAST_GATES), "{err}");
assert_eq!(protos(&qa), vec!["zk-quic"]);
}
#[test]
fn a_v2_server_with_the_wrong_key_is_refused_by_the_client() {
let (qa, _qb, addr, _b) = v2_pair(true, true, UNSET);
qa.fingerprint_hints
.insert(addr, NodeKey::generate().fingerprint());
let err = qa.offer_task(addr, &old_offer()).unwrap_err();
assert!(err.contains("handshake"), "{err}");
assert!(protos(&qa).is_empty());
}
#[test]
fn a_v2_client_falls_back_to_a_legacy_only_server() {
let a = keyed_state();
let qa = QuicTransport::start(lo(), a, true).unwrap();
let (c, k) = super::derive_quic_identity(MESH_KEY).unwrap();
let cfg = super::make_server_config(None, Some((c, k))).unwrap();
let addr = {
let _g = qa.runtime.enter();
let ep = quinn::Endpoint::server(cfg, lo()).unwrap();
let addr = ep.local_addr().unwrap();
qa.runtime.spawn(async move {
while let Some(i) = ep.accept().await {
tokio::spawn(async move {
if let Ok(c) = i.await {
c.closed().await;
}
});
}
});
addr
};
qa.fingerprint_hints
.insert(addr, NodeKey::generate().fingerprint());
qa.runtime.block_on(qa.get_or_connect(addr)).unwrap();
assert_eq!(protos(&qa), vec!["zk-quic"]);
let b = keyed_state();
let qb = QuicTransport::start(lo(), b, false).unwrap();
qb.fingerprint_hints
.insert(addr, NodeKey::generate().fingerprint());
let err = qb.runtime.block_on(qb.get_or_connect(addr)).unwrap_err();
assert!(err.contains("legacy zk-quic is off"), "{err}");
}
#[test]
fn a_server_with_legacy_off_refuses_zk_quic() {
let a = keyed_state();
let (b, _) = receiver(&a.node_key);
let qa = QuicTransport::start(lo(), a, true).unwrap();
let qb = QuicTransport::start(lo(), b.clone(), false).unwrap();
let addr = qb.local_addr().unwrap();
let err = qa.runtime.block_on(qa.get_or_connect(addr)).unwrap_err();
assert!(err.contains("handshake"), "{err}");
qa.fingerprint_hints.insert(addr, b.node_key.fingerprint());
qa.runtime.block_on(qa.get_or_connect(addr)).unwrap();
assert_eq!(protos(&qa), vec!["zk-quic/2"]);
}
#[test]
fn a_keyless_broker_refuses_legacy_quic_but_serves_v2() {
let keyless = Arc::new(BrokerState::with_config(BrokerConfig {
peer_key: None,
..Default::default()
}));
assert!(keyless.peer_manager.peer_key().is_empty());
let a = keyed_state();
keyless.node_roster.insert(a.node_key.public_b64(), false);
let qk = QuicTransport::start(lo(), keyless.clone(), true).unwrap();
assert!(qk.legacy_client.is_none(), "keyless must not dial legacy");
let addr = qk.local_addr().unwrap();
let qa = QuicTransport::start(lo(), a, true).unwrap();
let err = qa.runtime.block_on(qa.get_or_connect(addr)).unwrap_err();
assert!(err.contains("handshake"), "legacy served keyless: {err}");
qa.fingerprint_hints
.insert(addr, keyless.node_key.fingerprint());
qa.runtime.block_on(qa.get_or_connect(addr)).unwrap();
assert_eq!(protos(&qa), vec!["zk-quic/2"]);
}
#[test]
fn a_v2_connection_stands_in_for_the_node_signature_on_the_wire() {
let unsigned = |qa: &QuicTransport, addr| {
qa.runtime.block_on(async {
let conn = qa.get_or_connect(addr).await.unwrap();
let (mut s, mut r) = conn.open_bi().await.unwrap();
let mut o = old_offer();
o.peer_key = MESH_KEY.into();
super::write_msg(&mut s, &serde_json::to_vec(&o).unwrap())
.await
.unwrap();
String::from_utf8_lossy(&super::read_msg(&mut r).await.unwrap()).to_string()
})
};
let (qa, _qb, addr, _b) = v2_pair(true, true, SIG_ONLY);
let reply = unsigned(&qa, addr);
assert!(reply.contains(PAST_GATES), "{reply}");
let (qc, _qd, addr, _d) = v2_pair(true, true, SIG_ONLY);
qc.fingerprint_hints.clear(); let reply = unsigned(&qc, addr);
assert!(reply.contains("node signature check failed"), "{reply}");
}
}