1use std::collections::HashSet;
9use std::io::{Read, Write};
10use std::net::{SocketAddr, TcpListener, TcpStream};
11use std::path::{Path, PathBuf};
12use std::sync::{Arc, Mutex, PoisonError};
13use std::time::{Duration, Instant};
14
15use dive_core::owned_exchange::{Exchange, Startup};
16use dive_core::{Error, Handle, Heap, Machine, Opts, Plan, Store};
17use hmac::{Hmac, KeyInit, Mac};
18use serde::{Deserialize, Serialize};
19use sha2::{Digest, Sha256};
20
21mod wire;
22
23pub const MAX_FRAME_BYTES: usize = 64 << 20;
25pub const MAX_DAG_BYTES: usize = 32 << 20;
27pub const MAX_SUCCESSORS: u64 = 16_384;
29const KEY_BYTES: usize = 1 << 20;
30const MAGIC: &[u8; 8] = b"DIVEWN01";
31const DOMAIN: &[u8] = b"dive worker HMAC-SHA256 v1\0";
32const ACCEPT_POLL_MIN: Duration = Duration::from_micros(50);
34const ACCEPT_POLL_MAX: Duration = Duration::from_millis(10);
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38struct AcceptBackoff(Duration);
39
40impl Default for AcceptBackoff {
41 fn default() -> Self {
42 Self(ACCEPT_POLL_MIN)
43 }
44}
45
46impl AcceptBackoff {
47 fn next_wait(&mut self) -> Duration {
49 let wait = self.0;
50 self.0 = (wait * 2).min(ACCEPT_POLL_MAX);
51 wait
52 }
53
54 fn reset(&mut self) {
56 self.0 = ACCEPT_POLL_MIN;
57 }
58}
59
60const PING: u8 = 1;
61const PONG: u8 = 2;
62const WORK: u8 = 3;
63const QUERY: u8 = 4;
64const ANSWER: u8 = 5;
65const RESULT: u8 = 6;
66const FAILED: u8 = 7;
67
68fn failure(message: impl std::fmt::Display) -> Error {
69 Error::internal(format!("DIVE-S0500: distributed safety: {message}"))
70}
71fn io(error: impl std::fmt::Display) -> Error {
72 Error::internal(format!("DIVE-S0502: distributed I/O: {error}"))
73}
74fn random() -> Result<[u8; 32], Error> {
75 let mut value = [0; 32];
76 getrandom::fill(&mut value).map_err(failure)?;
77 Ok(value)
78}
79
80fn remaining(deadline: Instant) -> Result<Duration, Error> {
81 deadline
82 .checked_duration_since(Instant::now())
83 .filter(|n| !n.is_zero())
84 .ok_or_else(|| {
85 io(std::io::Error::new(
86 std::io::ErrorKind::TimedOut,
87 "authenticated unit deadline",
88 ))
89 })
90}
91fn timed_out(error: &std::io::Error) -> String {
93 if matches!(
94 error.kind(),
95 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
96 ) {
97 "no reply within the --worker-timeout-ms deadline".into()
98 } else {
99 error.to_string()
100 }
101}
102fn read_until(
105 stream: &mut TcpStream,
106 mut bytes: &mut [u8],
107 deadline: Instant,
108) -> Result<(), Error> {
109 while !bytes.is_empty() {
110 stream
111 .set_read_timeout(Some(remaining(deadline)?))
112 .map_err(io)?;
113 match stream.read(bytes) {
114 Ok(0) => {
115 return Err(io(std::io::Error::new(
116 std::io::ErrorKind::UnexpectedEof,
117 "incomplete authenticated frame",
118 )));
119 }
120 Ok(n) => bytes = &mut bytes[n..],
121 Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
122 Err(e) => return Err(io(timed_out(&e))),
123 }
124 }
125 Ok(())
126}
127fn write_until(stream: &mut TcpStream, mut bytes: &[u8], deadline: Instant) -> Result<(), Error> {
128 while !bytes.is_empty() {
129 stream
130 .set_write_timeout(Some(remaining(deadline)?))
131 .map_err(io)?;
132 match stream.write(bytes) {
133 Ok(0) => {
134 return Err(io(std::io::Error::new(
135 std::io::ErrorKind::WriteZero,
136 "incomplete authenticated frame",
137 )));
138 }
139 Ok(n) => bytes = &bytes[n..],
140 Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
141 Err(e) => return Err(io(timed_out(&e))),
142 }
143 }
144 Ok(())
145}
146fn hex(bytes: &[u8]) -> String {
147 use std::fmt::Write as _;
148 bytes
149 .iter()
150 .fold(String::with_capacity(bytes.len() * 2), |mut text, byte| {
151 let _ = write!(&mut text, "{byte:02x}");
152 text
153 })
154}
155
156struct Secret([u8; 32]);
157impl Secret {
158 fn read(path: &Path) -> Result<Self, Error> {
159 let mut bytes = Vec::new();
160 let named = |e: std::io::Error| io(format!("worker key file {}: {e}", path.display()));
161 std::fs::File::open(path)
162 .map_err(named)?
163 .take(33)
164 .read_to_end(&mut bytes)
165 .map_err(named)?;
166 let key = bytes.try_into().map_err(|_| {
167 failure(format!(
168 "worker key file {} must contain exactly 32 bytes",
169 path.display()
170 ))
171 })?;
172 Ok(Self(key))
173 }
174 fn mac(&self, parts: &[&[u8]]) -> Result<Hmac<Sha256>, Error> {
175 let mut mac = <Hmac<Sha256> as KeyInit>::new_from_slice(&self.0).map_err(failure)?;
176 mac.update(DOMAIN);
177 for part in parts {
178 mac.update(part);
179 }
180 Ok(mac)
181 }
182 fn tag(&self, parts: &[&[u8]]) -> Result<Vec<u8>, Error> {
183 Ok(self.mac(parts)?.finalize().into_bytes().to_vec())
184 }
185 fn verify(&self, parts: &[&[u8]], tag: &[u8]) -> Result<(), Error> {
186 self.mac(parts)?
187 .verify_slice(tag)
188 .map_err(|_| failure("authentication failed"))
189 }
190}
191
192#[derive(Clone, Debug)]
195pub struct Options {
196 pub workers: Vec<SocketAddr>,
198 pub key_file: PathBuf,
200 pub timeout: Duration,
202 pub retries: u8,
204}
205impl Options {
206 #[must_use]
208 pub fn new(workers: Vec<SocketAddr>, key_file: PathBuf) -> Self {
209 Self {
210 workers,
211 key_file,
212 timeout: Duration::from_secs(5),
213 retries: 2,
214 }
215 }
216 fn validate(&self) -> Result<(), Error> {
217 let distinct: HashSet<_> = self.workers.iter().collect();
218 if distinct.len() != self.workers.len()
219 || !(2..=64).contains(&distinct.len())
220 || self.timeout.is_zero()
221 || self.timeout > Duration::from_secs(60)
222 || self.retries > 8
223 {
224 return Err(failure(
225 "require 2..64 distinct workers, nonzero <=60s unit timeout and <=8 retries",
226 ));
227 }
228 Ok(())
229 }
230}
231
232#[derive(Clone, Debug, Serialize, Deserialize)]
234pub struct WorkerInfo {
235 pub id: String,
237 pub process_id: u32,
239 pub sent_units: u64,
241 pub result_units: u64,
243}
244#[derive(Clone, Debug, Default, Serialize, Deserialize)]
246pub struct Usage {
247 pub configured_workers: usize,
249 pub workers: Vec<WorkerInfo>,
251 pub assigned_units: u64,
253 pub completed_units: u64,
255 pub retries: u64,
257 pub epoch: u64,
259 pub membership_queries: u64,
261 pub duplicate_frames: u64,
263 pub failed_units: u64,
265 pub failures: Vec<String>,
267}
268
269#[derive(Clone, Copy)]
270struct Ticket {
271 epoch: u64,
272 work: u64,
273 attempt: u64,
274}
275#[derive(Clone, Copy)]
276struct Binding {
277 identity: [u8; 32],
278 run: [u8; 32],
279 client: [u8; 32],
280 server: [u8; 32],
281 ticket: Ticket,
282}
283impl Binding {
284 fn hello(&self) -> Vec<u8> {
285 let mut bytes = Vec::with_capacity(128);
286 bytes.extend_from_slice(MAGIC);
287 bytes.extend_from_slice(&self.identity);
288 bytes.extend_from_slice(&self.run);
289 bytes.extend_from_slice(&self.client);
290 for n in [self.ticket.epoch, self.ticket.work, self.ticket.attempt] {
291 bytes.extend_from_slice(&n.to_le_bytes());
292 }
293 bytes
294 }
295 fn context(&self) -> Vec<u8> {
296 let mut bytes = self.hello();
297 bytes.extend_from_slice(&self.server);
298 bytes
299 }
300}
301
302struct Channel {
303 stream: TcpStream,
304 binding: Binding,
305 send: u64,
306 receive: u64,
307 coordinator: bool,
308 deadline: Instant,
309}
310impl Channel {
311 fn direction(coordinator: bool) -> &'static [u8] {
312 if coordinator {
313 b"coordinator frame"
314 } else {
315 b"worker frame"
316 }
317 }
318 fn write(&mut self, key: &Secret, kind: u8, payload: &[u8]) -> Result<(), Error> {
319 if payload.len() > MAX_FRAME_BYTES {
320 return Err(failure("frame byte admission"));
321 }
322 let mut header = [0; 17];
323 header[0] = kind;
324 header[1..9].copy_from_slice(&self.send.to_le_bytes());
325 header[9..].copy_from_slice(&(payload.len() as u64).to_le_bytes());
326 let tag = key.tag(&[
327 Self::direction(self.coordinator),
328 &self.binding.context(),
329 &header,
330 payload,
331 ])?;
332 write_until(&mut self.stream, &header, self.deadline)?;
333 write_until(&mut self.stream, payload, self.deadline)?;
334 write_until(&mut self.stream, &tag, self.deadline)?;
335 self.send = self
336 .send
337 .checked_add(1)
338 .ok_or_else(|| failure("frame sequence exhausted"))?;
339 Ok(())
340 }
341 fn read(&mut self, key: &Secret, usage: &Mutex<Usage>) -> Result<(u8, Vec<u8>), Error> {
342 for _ in 0..64 {
343 let mut header = [0; 17];
344 read_until(&mut self.stream, &mut header, self.deadline)?;
345 let seq = u64::from_le_bytes(header[1..9].try_into().map_err(failure)?);
346 let len = usize::try_from(u64::from_le_bytes(header[9..].try_into().map_err(failure)?))
347 .map_err(failure)?;
348 if len > MAX_FRAME_BYTES {
349 return Err(failure("frame byte admission"));
350 }
351 let mut payload = Vec::new();
352 payload.try_reserve_exact(len).map_err(failure)?;
353 payload.resize(len, 0);
354 read_until(&mut self.stream, &mut payload, self.deadline)?;
355 let mut tag = [0; 32];
356 read_until(&mut self.stream, &mut tag, self.deadline)?;
357 key.verify(
358 &[
359 Self::direction(!self.coordinator),
360 &self.binding.context(),
361 &header,
362 &payload,
363 ],
364 &tag,
365 )?;
366 if seq < self.receive {
367 usage
368 .lock()
369 .unwrap_or_else(PoisonError::into_inner)
370 .duplicate_frames += 1;
371 continue;
372 }
373 if seq != self.receive {
374 return Err(failure("future or stale session sequence"));
375 }
376 self.receive = self
377 .receive
378 .checked_add(1)
379 .ok_or_else(|| failure("frame sequence exhausted"))?;
380 return Ok((header[0], payload));
381 }
382 Err(failure("duplicate frame admission"))
383 }
384}
385
386fn timeouts(stream: &TcpStream, timeout: Duration) -> Result<(), Error> {
387 stream
388 .set_nodelay(true)
389 .and_then(|()| stream.set_read_timeout(Some(timeout)))
390 .and_then(|()| stream.set_write_timeout(Some(timeout)))
391 .map_err(io)
392}
393fn connect(
394 address: SocketAddr,
395 key: &Secret,
396 identity: [u8; 32],
397 run: [u8; 32],
398 ticket: Ticket,
399 timeout: Duration,
400) -> Result<(Channel, WorkerInfo), Error> {
401 let deadline = Instant::now() + timeout;
402 let mut stream = TcpStream::connect_timeout(&address, timeout).map_err(io)?;
403 timeouts(&stream, timeout)?;
404 let mut binding = Binding {
405 identity,
406 run,
407 client: random()?,
408 server: [0; 32],
409 ticket,
410 };
411 let hello = binding.hello();
412 let tag = key.tag(&[b"hello", &hello])?;
413 write_until(&mut stream, &hello, deadline)?;
414 write_until(&mut stream, &tag, deadline)?;
415 let mut ack = [0; 69];
416 let mut mac = [0; 32];
417 read_until(&mut stream, &mut ack, deadline)
420 .and_then(|()| read_until(&mut stream, &mut mac, deadline))
421 .map_err(|error| {
422 io(format!(
423 "worker {address} closed or did not answer the authenticated handshake \
424 (check that every role reads the same --worker-key file and that the \
425 worker is running and not stalled; a different dive build answers \
426 with an identity mismatch instead): {}",
427 error
428 .message()
429 .trim_start_matches("DIVE-S0502: distributed I/O: ")
430 ))
431 })?;
432 key.verify(&[b"ack", &hello, &ack], &mac)?;
433 if ack[0] != 0 {
434 return Err(failure(
435 "worker source/config/executable/policy/startup identity mismatch",
436 ));
437 }
438 binding.server.copy_from_slice(&ack[1..33]);
439 let info = WorkerInfo {
440 id: hex(&ack[33..65]),
441 process_id: u32::from_le_bytes(ack[65..].try_into().map_err(failure)?),
442 sent_units: 0,
443 result_units: 0,
444 };
445 Ok((
446 Channel {
447 stream,
448 binding,
449 send: 0,
450 receive: 0,
451 coordinator: true,
452 deadline,
453 },
454 info,
455 ))
456}
457fn accept(
458 mut stream: TcpStream,
459 key: &Secret,
460 identity: [u8; 32],
461 worker: [u8; 32],
462 timeout: Duration,
463) -> Result<Channel, Error> {
464 let deadline = Instant::now() + timeout;
465 stream.set_nonblocking(false).map_err(io)?;
469 timeouts(&stream, timeout)?;
470 let mut hello = [0; 128];
471 let mut tag = [0; 32];
472 read_until(&mut stream, &mut hello, deadline)?;
473 read_until(&mut stream, &mut tag, deadline)?;
474 key.verify(&[b"hello", &hello], &tag)?;
475 if &hello[..8] != MAGIC {
476 return Err(failure("worker protocol version"));
477 }
478 if hello[8..40] != identity {
479 let mut rejection = [0; 69];
480 rejection[0] = 1;
481 let tag = key.tag(&[b"ack", &hello, &rejection])?;
482 write_until(&mut stream, &rejection, deadline)?;
483 write_until(&mut stream, &tag, deadline)?;
484 return Err(failure(
485 "source/config/executable/policy/startup identity mismatch",
486 ));
487 }
488 let word = |at| -> Result<u64, Error> {
489 Ok(u64::from_le_bytes(
490 hello[at..at + 8].try_into().map_err(failure)?,
491 ))
492 };
493 let mut binding = Binding {
494 identity,
495 run: hello[40..72].try_into().map_err(failure)?,
496 client: hello[72..104].try_into().map_err(failure)?,
497 server: random()?,
498 ticket: Ticket {
499 epoch: word(104)?,
500 work: word(112)?,
501 attempt: word(120)?,
502 },
503 };
504 let mut ack = [0; 69];
505 ack[1..33].copy_from_slice(&binding.server);
506 ack[33..65].copy_from_slice(&worker);
507 ack[65..].copy_from_slice(&std::process::id().to_le_bytes());
508 let tag = key.tag(&[b"ack", &hello, &ack])?;
509 write_until(&mut stream, &ack, deadline)?;
510 write_until(&mut stream, &tag, deadline)?;
511 binding.identity = identity;
513 Ok(Channel {
514 stream,
515 binding,
516 send: 0,
517 receive: 0,
518 coordinator: false,
519 deadline,
520 })
521}
522
523fn identity(base: &[u8], store: &Store) -> Result<([u8; 32], Vec<u8>), Error> {
524 store.strings().seal();
525 let image = dive_core::store::snapshot::capture(store, MAX_DAG_BYTES, &|| true)?;
526 let mut hash = Sha256::new();
527 hash.update(b"dive distributed pure core 0151 v1\0");
528 hash.update((base.len() as u64).to_le_bytes());
529 hash.update(base);
530 hash.update(&image);
531 Ok((hash.finalize().into(), image))
532}
533
534pub struct Coordinator {
536 options: Options,
537 key: Secret,
538 identity: [u8; 32],
539 run: [u8; 32],
540 startup: Startup,
541 usage: Mutex<Usage>,
542}
543
544pub(crate) struct Received {
547 exchange: Exchange,
548 payload: Result<Vec<u8>, Error>,
549 epoch: u64,
550}
551impl Coordinator {
552 pub fn new(options: Options, base_identity: &[u8], store: &Arc<Store>) -> Result<Self, Error> {
557 options.validate()?;
558 let key = Secret::read(&options.key_file)?;
559 let (identity, _) = identity(base_identity, store)?;
560 let run = random()?;
561 let startup = Startup::new(
562 &Heap::new(Arc::clone(store), true, false),
563 MAX_DAG_BYTES,
564 &|| true,
565 )?;
566 let owner = Self {
567 usage: Mutex::new(Usage {
568 configured_workers: options.workers.len(),
569 ..Usage::default()
570 }),
571 options,
572 key,
573 identity,
574 run,
575 startup,
576 };
577 for &address in &owner.options.workers {
578 let (mut channel, info) = connect(
579 address,
580 &owner.key,
581 identity,
582 run,
583 Ticket {
584 epoch: 0,
585 work: u64::MAX,
586 attempt: 0,
587 },
588 owner.options.timeout,
589 )?;
590 channel.write(&owner.key, PING, &[])?;
591 if channel.read(&owner.key, &owner.usage)?.0 != PONG {
592 return Err(failure("startup acknowledgement"));
593 }
594 owner.worker(info);
595 }
596 if owner.usage().workers.len() < 2 {
597 return Err(failure("two independently initialized workers required"));
598 }
599 Ok(owner)
600 }
601 fn worker(&self, info: WorkerInfo) {
602 let mut usage = self.usage.lock().unwrap_or_else(PoisonError::into_inner);
603 if !usage.workers.iter().any(|w| w.id == info.id) {
604 usage.workers.push(info);
605 }
606 }
607 fn unit_fact(&self, id: &str, result: bool) -> Result<(), Error> {
608 let mut usage = self.usage.lock().unwrap_or_else(PoisonError::into_inner);
609 let info = usage
610 .workers
611 .iter_mut()
612 .find(|worker| worker.id == id)
613 .ok_or_else(|| failure("unregistered authenticated worker"))?;
614 let count = if result {
615 &mut info.result_units
616 } else {
617 &mut info.sent_units
618 };
619 *count = count
620 .checked_add(1)
621 .ok_or_else(|| failure("worker unit counter overflow"))?;
622 Ok(())
623 }
624 #[must_use]
626 pub fn usage(&self) -> Usage {
627 self.usage
628 .lock()
629 .unwrap_or_else(PoisonError::into_inner)
630 .clone()
631 }
632 pub(crate) fn record_failure(&self, error: &Error) {
633 if error.message().starts_with("DIVE-S05") {
634 let mut facts = self.usage.lock().unwrap_or_else(PoisonError::into_inner);
635 if !facts
636 .failures
637 .iter()
638 .any(|failure| failure == error.message())
639 {
640 facts.failures.push(error.message().into());
641 }
642 }
643 }
644 pub(crate) fn width(&self) -> usize {
645 self.options.workers.len()
646 }
647
648 #[allow(
649 clippy::too_many_arguments,
650 reason = "one bounded batch borrows the actual committed seen authority"
651 )]
652 pub(crate) fn expand(
653 &self,
654 plan: &Plan,
655 store: &Arc<Store>,
656 rows: &[Handle],
657 seen: &crate::Seen,
658 epoch: u64,
659 poll: &(dyn Fn() -> bool + Sync),
660 ) -> Result<Vec<Option<Received>>, Error> {
661 let width = plan.vars.len();
662 if width == 0 || !rows.len().is_multiple_of(width) || rows.len() / width > self.width() {
663 return Err(failure("remote batch dimensions"));
664 }
665 let heap = Heap::new(Arc::clone(store), true, false);
666 let mut jobs = Vec::new();
667 jobs.try_reserve_exact(rows.len() / width)
668 .map_err(failure)?;
669 for row in rows.chunks_exact(width) {
670 jobs.push(Exchange::request_with_startup(
671 &heap,
672 row,
673 MAX_DAG_BYTES,
674 &|| !poll(),
675 &self.startup,
676 )?);
677 }
678 let first = {
679 let mut facts = self.usage.lock().unwrap_or_else(PoisonError::into_inner);
680 let first = facts.assigned_units;
681 facts.assigned_units = facts
682 .assigned_units
683 .checked_add(jobs.len() as u64)
684 .ok_or_else(|| failure("work ids exhausted"))?;
685 first
686 };
687 let received = std::thread::scope(|scope| {
688 let handles: Vec<_> = jobs
689 .into_iter()
690 .enumerate()
691 .map(|(i, (bytes, exchange))| {
692 std::thread::Builder::new()
693 .name("dive-network".into())
694 .spawn_scoped(scope, move || {
695 (
696 exchange,
697 self.job(first + i as u64, epoch, &bytes, seen, store, poll),
698 )
699 })
700 .map_err(|error| failure(format!("network thread start: {error}")))
701 })
702 .collect::<Result<Vec<_>, _>>()?;
703 handles
704 .into_iter()
705 .map(|h| {
706 h.join()
707 .map_err(|_| failure("network worker thread failed"))
708 })
709 .collect::<Result<Vec<_>, _>>()
710 })?;
711 let mut result = Vec::new();
712 result.try_reserve_exact(received.len()).map_err(failure)?;
713 for (exchange, payload) in received {
714 result.push(Some(Received {
715 exchange,
716 payload,
717 epoch,
718 }));
719 }
720 Ok(result)
721 }
722
723 pub(crate) fn import(
724 &self,
725 received: Received,
726 plan: &Plan,
727 heap: &mut Heap,
728 poll: &(dyn Fn() -> bool + Sync),
729 metrics: &mut crate::Metrics,
730 ) -> Result<crate::parallel::Expanded, Error> {
731 let payload = received.payload?;
732 let (meta, dag) = wire::split(&payload)?;
733 let roots = received
734 .exchange
735 .receive_reply(heap, dag, MAX_DAG_BYTES, &|| !poll())?;
736 let (expanded, counts) = wire::decode(
737 meta,
738 &roots,
739 plan,
740 usize::try_from(MAX_SUCCESSORS).map_err(failure)?,
741 )?;
742 counts.apply(metrics)?;
743 let mut facts = self.usage.lock().unwrap_or_else(PoisonError::into_inner);
744 facts.completed_units += 1;
745 facts.epoch = facts.epoch.max(received.epoch);
746 Ok(expanded)
747 }
748
749 fn job(
750 &self,
751 work: u64,
752 epoch: u64,
753 bytes: &[u8],
754 seen: &crate::Seen,
755 store: &Arc<Store>,
756 poll: &(dyn Fn() -> bool + Sync),
757 ) -> Result<Vec<u8>, Error> {
758 let mut final_error = failure("work was not attempted");
759 for attempt in 0..=self.options.retries {
760 if poll() {
761 return Err(Error::stop());
762 }
763 if attempt > 0 {
764 self.usage
765 .lock()
766 .unwrap_or_else(PoisonError::into_inner)
767 .retries += 1;
768 }
769 let index = usize::try_from(work % (self.width() as u64)).map_err(failure)?;
770 let address = self.options.workers[(index + usize::from(attempt)) % self.width()];
771 let result = (|| {
772 let (mut channel, info) = connect(
773 address,
774 &self.key,
775 self.identity,
776 self.run,
777 Ticket {
778 epoch,
779 work,
780 attempt: u64::from(attempt),
781 },
782 self.options.timeout,
783 )?;
784 let worker_id = info.id.clone();
785 self.worker(info);
786 channel.write(&self.key, WORK, bytes)?;
787 self.unit_fact(&worker_id, false)?;
788 let heap = Heap::new(Arc::clone(store), true, false);
789 loop {
790 if poll() {
791 return Err(Error::stop());
792 }
793 let (kind, payload) = channel.read(&self.key, &self.usage)?;
794 match kind {
795 QUERY => {
796 if payload.len() < 8 || payload.len() > 8 + KEY_BYTES {
797 return Err(failure("membership query admission"));
798 }
799 let key = u64::from_le_bytes(payload[..8].try_into().map_err(failure)?);
800 let present = if let Some(exact) = &seen.exact {
801 exact.contains_encoded(
802 key,
803 &payload[8..],
804 &heap,
805 KEY_BYTES,
806 poll,
807 )?
808 } else {
809 seen.contains(key)
810 };
811 self.usage
812 .lock()
813 .unwrap_or_else(PoisonError::into_inner)
814 .membership_queries += 1;
815 channel.write(&self.key, ANSWER, &[u8::from(present)])?;
816 }
817 RESULT => {
818 self.unit_fact(&worker_id, true)?;
819 return Ok(payload);
820 }
821 FAILED => {
822 return Err(failure(format!(
823 "worker refused incomplete unit: {}",
824 String::from_utf8_lossy(&payload)
825 )));
826 }
827 _ => return Err(failure("unexpected worker message")),
828 }
829 }
830 })();
831 match result {
832 Ok(bytes) => return Ok(bytes),
833 Err(error) if error.kind() == dive_core::ErrorKind::Stop => return Err(error),
834 Err(error) => {
835 let retry = error.message().starts_with("DIVE-S0502:");
836 final_error = error;
837 if !retry {
838 break;
839 }
840 }
841 }
842 }
843 self.usage
844 .lock()
845 .unwrap_or_else(PoisonError::into_inner)
846 .failed_units += 1;
847 Err(final_error)
848 }
849}
850
851pub struct Worker {
853 plan: Plan,
854 store: Arc<Store>,
855 startup: Vec<u8>,
856 opts: Opts,
857 key: Secret,
858 identity: [u8; 32],
859 id: [u8; 32],
860 timeout: Duration,
861}
862impl Worker {
863 pub fn new(
868 plan: Plan,
869 store: Arc<Store>,
870 opts: Opts,
871 base_identity: &[u8],
872 key_file: &Path,
873 timeout: Duration,
874 ) -> Result<Self, Error> {
875 if plan.effects.any || plan.view.is_some() || plan.symmetry.is_some() {
876 return Err(Error::unsupported("distributed effects, VIEW or symmetry"));
877 }
878 if timeout.is_zero() || timeout > Duration::from_secs(60) {
879 return Err(failure("worker timeout must be nonzero and <=60s"));
880 }
881 let (identity, startup) = identity(base_identity, &store)?;
882 Ok(Self {
883 plan,
884 store,
885 startup,
886 opts,
887 key: Secret::read(key_file)?,
888 identity,
889 id: random()?,
890 timeout,
891 })
892 }
893 pub fn serve(
899 &self,
900 listener: &TcpListener,
901 max_units: Option<u64>,
902 stop: &(dyn Fn() -> bool + Sync),
903 ) -> Result<Usage, Error> {
904 listener.set_nonblocking(true).map_err(io)?;
905 let usage = Mutex::new(Usage::default());
906 let mut idle = AcceptBackoff::default();
910 while !stop()
911 && max_units.is_none_or(|n| {
912 usage
913 .lock()
914 .unwrap_or_else(PoisonError::into_inner)
915 .completed_units
916 < n
917 })
918 {
919 let stream = match listener.accept() {
920 Ok((s, _)) => {
921 idle.reset();
922 s
923 }
924 Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
925 std::thread::sleep(idle.next_wait());
926 continue;
927 }
928 Err(e) => return Err(io(e)),
929 };
930 let result = accept(stream, &self.key, self.identity, self.id, self.timeout).and_then(
931 |mut channel| {
932 let result = self.unit(&mut channel, &usage);
933 if let Err(error) = &result {
934 let _ = channel.write(&self.key, FAILED, error.message().as_bytes());
935 }
936 result
937 },
938 );
939 if let Err(error) = result {
940 let mut facts = usage.lock().unwrap_or_else(PoisonError::into_inner);
941 if facts.failures.len() < 64 {
942 facts.failures.push(error.message().into());
943 }
944 }
945 }
946 Ok(usage.into_inner().unwrap_or_else(PoisonError::into_inner))
947 }
948 fn unit(&self, channel: &mut Channel, usage: &Mutex<Usage>) -> Result<(), Error> {
949 let (kind, payload) = channel.read(&self.key, usage)?;
950 if kind == PING && payload.is_empty() {
951 return channel.write(&self.key, PONG, &[]);
952 }
953 if kind != WORK {
954 return Err(failure("work kind"));
955 }
956 let deadline = channel.deadline;
957 let poll = || Instant::now() >= deadline;
958 let work = (|| {
959 let store = dive_core::store::snapshot::restore(
960 &self.store,
961 &self.startup,
962 &self.startup,
963 MAX_DAG_BYTES,
964 &|| !poll(),
965 )?;
966 let mut machine = Machine::new(
967 store,
968 self.plan.vars.len(),
969 self.opts,
970 dive_core::memo::Memo::new(
971 &self
972 .plan
973 .code
974 .sites
975 .iter()
976 .map(dive_core::plan::MemoSite::key_len)
977 .collect::<Vec<_>>(),
978 ),
979 );
980 machine.set_work_control(Some(Arc::new(Deadline(deadline))));
981 let startup = Startup::new(&machine.heap, MAX_DAG_BYTES, &|| !poll())?;
982 let (source, exchange) = Exchange::receive_request_with_startup(
983 &mut machine.heap,
984 &payload,
985 MAX_DAG_BYTES,
986 &|| !poll(),
987 &startup,
988 )?;
989 if source.len() != self.plan.vars.len() || source.iter().any(|h| h.is_empty()) {
990 return Err(failure("source row width/completeness"));
991 }
992 let mut member = RemoteMember {
993 channel,
994 key: &self.key,
995 usage,
996 };
997 let mut metrics = crate::Metrics::default();
998 let expanded = crate::parallel::expand_remote(
999 &mut machine,
1000 &self.plan,
1001 &source,
1002 MAX_SUCCESSORS,
1003 &poll,
1004 &mut metrics,
1005 &mut member,
1006 );
1007 let (meta, roots) = wire::encode(&expanded, &metrics, MAX_FRAME_BYTES)?;
1008 let dag = exchange.reply(&machine.heap, &roots, MAX_DAG_BYTES, &|| !poll())?;
1009 wire::join(&meta, &dag, MAX_FRAME_BYTES)
1010 })();
1011 match work {
1012 Ok(bytes) => {
1013 channel.write(&self.key, RESULT, &bytes)?;
1014 usage
1015 .lock()
1016 .unwrap_or_else(PoisonError::into_inner)
1017 .completed_units += 1;
1018 Ok(())
1019 }
1020 Err(error) => {
1021 let mut facts = usage.lock().unwrap_or_else(PoisonError::into_inner);
1022 if facts.failures.len() < 64 {
1023 facts.failures.push(error.message().into());
1024 }
1025 drop(facts);
1026 channel.deadline = Instant::now() + self.timeout;
1029 let message = if Instant::now() >= deadline {
1030 format!(
1031 "the unit did not finish within the worker's --worker-timeout-ms of {} ms",
1032 self.timeout.as_millis()
1033 )
1034 } else {
1035 error.message().to_owned()
1036 };
1037 channel.write(&self.key, FAILED, message.as_bytes())
1038 }
1039 }
1040 }
1041}
1042
1043#[derive(Debug)]
1044struct Deadline(Instant);
1045impl dive_core::eval::WorkControl for Deadline {
1046 fn proceed(&self) -> bool {
1047 Instant::now() < self.0
1048 }
1049}
1050
1051struct RemoteMember<'a> {
1052 channel: &'a mut Channel,
1053 key: &'a Secret,
1054 usage: &'a Mutex<Usage>,
1055}
1056impl crate::parallel::Membership for RemoteMember<'_> {
1057 fn contains(&mut self, key: u64, identity: &[Handle], heap: &Heap) -> Result<bool, Error> {
1058 let bytes = dive_core::key::canonical(heap, identity, KEY_BYTES, &|| {
1059 Instant::now() < self.channel.deadline
1060 })
1061 .map_err(|e| failure(format!("canonical worker key: {e:?}")))?;
1062 let mut query = Vec::new();
1063 query.extend_from_slice(&key.to_le_bytes());
1064 query.extend_from_slice(&bytes);
1065 self.channel.write(self.key, QUERY, &query)?;
1066 let (kind, answer) = self.channel.read(self.key, self.usage)?;
1067 if kind != ANSWER {
1068 return Err(failure("membership answer kind"));
1069 }
1070 match answer.as_slice() {
1071 [0] => Ok(false),
1072 [1] => Ok(true),
1073 _ => Err(failure("membership answer")),
1074 }
1075 }
1076}
1077
1078#[cfg(test)]
1079mod fault_tests;
1080
1081#[cfg(test)]
1082mod tests {
1083 #![allow(
1084 clippy::expect_used,
1085 reason = "local authenticated worker backend controls"
1086 )]
1087 use super::*;
1088 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
1089
1090 struct Stop(Arc<AtomicBool>);
1091 impl Drop for Stop {
1092 fn drop(&mut self) {
1093 self.0.store(true, Ordering::Relaxed);
1094 }
1095 }
1096
1097 #[test]
1098 fn accept_backoff_doubles_to_the_cap_and_resets_after_a_session() {
1099 let mut backoff = AcceptBackoff::default();
1100 let waits: Vec<Duration> = (0..12).map(|_| backoff.next_wait()).collect();
1101 assert_eq!(waits[0], ACCEPT_POLL_MIN);
1102 assert_eq!(waits[1], ACCEPT_POLL_MIN * 2);
1103 assert!(waits.windows(2).all(|w| w[1] >= w[0]));
1104 assert!(waits.iter().all(|w| *w <= ACCEPT_POLL_MAX));
1105 assert_eq!(waits[8], ACCEPT_POLL_MAX);
1107 assert_eq!(waits[11], ACCEPT_POLL_MAX);
1108 backoff.reset();
1109 assert_eq!(backoff.next_wait(), ACCEPT_POLL_MIN);
1110 }
1111
1112 #[test]
1115 fn accepted_session_waits_for_a_late_hello_from_a_nonblocking_listener() {
1116 let listener = TcpListener::bind("127.0.0.1:0").expect("local listener");
1117 listener
1118 .set_nonblocking(true)
1119 .expect("nonblocking listener");
1120 let address = listener.local_addr().expect("address");
1121 let client = std::thread::spawn(move || {
1122 let mut stream = TcpStream::connect(address).expect("connect");
1123 std::thread::sleep(Duration::from_millis(100));
1124 stream.write_all(&[0; 160]).expect("late hello");
1125 stream
1126 });
1127 let stream = loop {
1128 match listener.accept() {
1129 Ok((stream, _)) => break stream,
1130 Err(e) => {
1131 assert_eq!(e.kind(), std::io::ErrorKind::WouldBlock, "accept: {e}");
1132 std::thread::sleep(Duration::from_millis(1));
1133 }
1134 }
1135 };
1136 let error = accept(
1137 stream,
1138 &Secret([37; 32]),
1139 [0; 32],
1140 [0; 32],
1141 Duration::from_secs(5),
1142 )
1143 .err()
1144 .expect("an unauthenticated hello is rejected");
1145 assert_eq!(
1147 error.message(),
1148 "DIVE-S0500: distributed safety: authentication failed"
1149 );
1150 drop(client.join().expect("client"));
1151 }
1152
1153 #[test]
1154 fn two_independent_native_workers_complete_actual_ordered_search() {
1155 static SERIAL: AtomicU64 = AtomicU64::new(0);
1156 for exact in [false, true] {
1157 let path = std::env::temp_dir().join(format!(
1158 "dive-worker-key-{}-{}",
1159 std::process::id(),
1160 SERIAL.fetch_add(1, Ordering::Relaxed)
1161 ));
1162 std::fs::write(&path, [37; 32]).expect("test key");
1163 let store = Arc::new(Store::new(true));
1164 let plan = crate::tests::frontier_plan(&store, false).expect("coordinator plan");
1165 let local = crate::explore_with_storage(
1166 &plan,
1167 &store,
1168 crate::Config {
1169 opts: Opts::default(),
1170 max_states: None,
1171 max_depth: None,
1172 workers: 1,
1173 warn: true,
1174 sample: None,
1175 },
1176 &|| false,
1177 None,
1178 None,
1179 &crate::storage::Storage {
1180 exact,
1181 ..crate::storage::Storage::default()
1182 },
1183 );
1184 let mut workers = Vec::new();
1185 let mut listeners = Vec::new();
1186 let mut endpoints = Vec::new();
1187 for _ in 0..2 {
1188 let listener = TcpListener::bind("127.0.0.1:0").expect("local listener");
1189 endpoints.push(listener.local_addr().expect("address"));
1190 let owner = Arc::new(Store::new(true));
1191 let plan = crate::tests::frontier_plan(&owner, false).expect("independent plan");
1192 workers.push(
1193 Worker::new(
1194 plan,
1195 owner,
1196 Opts::default(),
1197 b"same independently compiled test",
1198 &path,
1199 Duration::from_secs(5),
1200 )
1201 .expect("worker"),
1202 );
1203 listeners.push(listener);
1204 }
1205 let stop = Arc::new(AtomicBool::new(false));
1206 std::thread::scope(|scope| {
1207 let _guard = Stop(Arc::clone(&stop));
1208 for (worker, listener) in workers.iter().zip(&listeners) {
1209 let stop = &stop;
1210 scope.spawn(move || {
1211 worker
1212 .serve(listener, None, &|| stop.load(Ordering::Relaxed))
1213 .expect("serve")
1214 });
1215 }
1216 let coordinator = Coordinator::new(
1217 Options::new(endpoints, path.clone()),
1218 b"same independently compiled test",
1219 &store,
1220 )
1221 .expect("authenticated coordinator");
1222 let out = crate::explore_distributed(
1223 &plan,
1224 &store,
1225 crate::Config {
1226 opts: Opts::default(),
1227 max_states: None,
1228 max_depth: None,
1229 workers: 1,
1230 warn: true,
1231 sample: None,
1232 },
1233 &|| false,
1234 None,
1235 &crate::storage::Storage {
1236 exact,
1237 ..crate::storage::Storage::default()
1238 },
1239 &coordinator,
1240 );
1241 assert_eq!(out.verdict, local.verdict, "{:?}", out.verdict);
1242 assert_eq!(out.stats, local.stats);
1243 let facts = coordinator.usage();
1244 assert_eq!(facts.workers.len(), 2);
1245 assert_eq!(facts.completed_units, 128);
1246 assert_eq!(facts.assigned_units, 128);
1247 assert_eq!(facts.membership_queries, 128);
1248 assert_eq!(facts.failures, [] as [String; 0]);
1249 });
1250 std::fs::remove_file(path).expect("remove key");
1251 }
1252 }
1253
1254 #[test]
1255 fn authentication_binds_identity_run_epoch_work_attempt_and_payload() {
1256 let key = Secret([17; 32]);
1257 let binding = Binding {
1258 identity: [1; 32],
1259 run: [2; 32],
1260 client: [3; 32],
1261 server: [4; 32],
1262 ticket: Ticket {
1263 epoch: 5,
1264 work: 6,
1265 attempt: 7,
1266 },
1267 };
1268 let context = binding.context();
1269 let tag = key
1270 .tag(&[&context, b"work", b"actual payload"])
1271 .expect("tag");
1272 assert!(
1273 key.verify(&[&context, b"work", b"actual payload"], &tag)
1274 .is_ok()
1275 );
1276 for at in [8, 40, 72, 104, 112, 120, 128] {
1277 let mut altered = context.clone();
1278 altered[at] ^= 1;
1279 assert!(
1280 key.verify(&[&altered, b"work", b"actual payload"], &tag)
1281 .is_err()
1282 );
1283 }
1284 assert!(
1285 key.verify(&[&context, b"result", b"actual payload"], &tag)
1286 .is_err()
1287 );
1288 assert!(
1289 key.verify(&[&context, b"work", b"altered payload"], &tag)
1290 .is_err()
1291 );
1292 }
1293}