Skip to main content

dive_explore/
distributed.rs

1//! Authenticated, bounded native pure-safety work over trusted TCP endpoints.
2//!
3//! HMAC authenticates identity and integrity; it does not encrypt traffic.
4//! Operators supply a private network or tunnel when confidentiality is needed.
5//! Received units contain values for an already compiled plan, never programs,
6//! executable paths, host grants or model-selection instructions.
7
8use std::collections::HashSet;
9use std::io::{Read, Write};
10use std::net::{SocketAddr, TcpListener, TcpStream};
11use std::path::{Path, PathBuf};
12use std::sync::{Arc, Mutex, PoisonError};
13use std::time::{Duration, Instant};
14
15use dive_core::owned_exchange::{Exchange, Startup};
16use dive_core::{Error, Handle, Heap, Machine, Opts, Plan, Store};
17use hmac::{Hmac, KeyInit, Mac};
18use serde::{Deserialize, Serialize};
19use sha2::{Digest, Sha256};
20
21mod wire;
22
23/// Private protocol frame admission, independent of ordinary local checking.
24pub const MAX_FRAME_BYTES: usize = 64 << 20;
25/// Private owned-value DAG admission per work or result.
26pub const MAX_DAG_BYTES: usize = 32 << 20;
27/// Actual source-emission admission; crossing it cannot certify completion.
28pub const MAX_SUCCESSORS: u64 = 16_384;
29const KEY_BYTES: usize = 1 << 20;
30const MAGIC: &[u8; 8] = b"DIVEWN01";
31const DOMAIN: &[u8] = b"dive worker HMAC-SHA256 v1\0";
32/// Shortest and longest wait between accept polls on an idle worker listener.
33const ACCEPT_POLL_MIN: Duration = Duration::from_micros(50);
34const ACCEPT_POLL_MAX: Duration = Duration::from_millis(10);
35/// Wait between accept polls: starts at [`ACCEPT_POLL_MIN`] after a session
36/// and doubles on each empty poll up to [`ACCEPT_POLL_MAX`].
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38struct AcceptBackoff(Duration);
39
40impl Default for AcceptBackoff {
41    fn default() -> Self {
42        Self(ACCEPT_POLL_MIN)
43    }
44}
45
46impl AcceptBackoff {
47    /// Returns the wait for this empty poll and doubles the next one, capped.
48    fn next_wait(&mut self) -> Duration {
49        let wait = self.0;
50        self.0 = (wait * 2).min(ACCEPT_POLL_MAX);
51        wait
52    }
53
54    /// Restarts from the shortest wait after a session arrives.
55    fn reset(&mut self) {
56        self.0 = ACCEPT_POLL_MIN;
57    }
58}
59
60const PING: u8 = 1;
61const PONG: u8 = 2;
62const WORK: u8 = 3;
63const QUERY: u8 = 4;
64const ANSWER: u8 = 5;
65const RESULT: u8 = 6;
66const FAILED: u8 = 7;
67
68fn failure(message: impl std::fmt::Display) -> Error {
69    Error::internal(format!("DIVE-S0500: distributed safety: {message}"))
70}
71fn io(error: impl std::fmt::Display) -> Error {
72    Error::internal(format!("DIVE-S0502: distributed I/O: {error}"))
73}
74fn random() -> Result<[u8; 32], Error> {
75    let mut value = [0; 32];
76    getrandom::fill(&mut value).map_err(failure)?;
77    Ok(value)
78}
79
80fn remaining(deadline: Instant) -> Result<Duration, Error> {
81    deadline
82        .checked_duration_since(Instant::now())
83        .filter(|n| !n.is_zero())
84        .ok_or_else(|| {
85            io(std::io::Error::new(
86                std::io::ErrorKind::TimedOut,
87                "authenticated unit deadline",
88            ))
89        })
90}
91// A socket timeout surfaces as WouldBlock on Unix; name the deadline instead.
92fn timed_out(error: &std::io::Error) -> String {
93    if matches!(
94        error.kind(),
95        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
96    ) {
97        "no reply within the --worker-timeout-ms deadline".into()
98    } else {
99        error.to_string()
100    }
101}
102// Refresh the remaining absolute deadline for every syscall. A peer trickling
103// bytes cannot extend a unit by repeatedly resetting a relative socket timeout.
104fn read_until(
105    stream: &mut TcpStream,
106    mut bytes: &mut [u8],
107    deadline: Instant,
108) -> Result<(), Error> {
109    while !bytes.is_empty() {
110        stream
111            .set_read_timeout(Some(remaining(deadline)?))
112            .map_err(io)?;
113        match stream.read(bytes) {
114            Ok(0) => {
115                return Err(io(std::io::Error::new(
116                    std::io::ErrorKind::UnexpectedEof,
117                    "incomplete authenticated frame",
118                )));
119            }
120            Ok(n) => bytes = &mut bytes[n..],
121            Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
122            Err(e) => return Err(io(timed_out(&e))),
123        }
124    }
125    Ok(())
126}
127fn write_until(stream: &mut TcpStream, mut bytes: &[u8], deadline: Instant) -> Result<(), Error> {
128    while !bytes.is_empty() {
129        stream
130            .set_write_timeout(Some(remaining(deadline)?))
131            .map_err(io)?;
132        match stream.write(bytes) {
133            Ok(0) => {
134                return Err(io(std::io::Error::new(
135                    std::io::ErrorKind::WriteZero,
136                    "incomplete authenticated frame",
137                )));
138            }
139            Ok(n) => bytes = &bytes[n..],
140            Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
141            Err(e) => return Err(io(timed_out(&e))),
142        }
143    }
144    Ok(())
145}
146fn hex(bytes: &[u8]) -> String {
147    use std::fmt::Write as _;
148    bytes
149        .iter()
150        .fold(String::with_capacity(bytes.len() * 2), |mut text, byte| {
151            let _ = write!(&mut text, "{byte:02x}");
152            text
153        })
154}
155
156struct Secret([u8; 32]);
157impl Secret {
158    fn read(path: &Path) -> Result<Self, Error> {
159        let mut bytes = Vec::new();
160        let named = |e: std::io::Error| io(format!("worker key file {}: {e}", path.display()));
161        std::fs::File::open(path)
162            .map_err(named)?
163            .take(33)
164            .read_to_end(&mut bytes)
165            .map_err(named)?;
166        let key = bytes.try_into().map_err(|_| {
167            failure(format!(
168                "worker key file {} must contain exactly 32 bytes",
169                path.display()
170            ))
171        })?;
172        Ok(Self(key))
173    }
174    fn mac(&self, parts: &[&[u8]]) -> Result<Hmac<Sha256>, Error> {
175        let mut mac = <Hmac<Sha256> as KeyInit>::new_from_slice(&self.0).map_err(failure)?;
176        mac.update(DOMAIN);
177        for part in parts {
178            mac.update(part);
179        }
180        Ok(mac)
181    }
182    fn tag(&self, parts: &[&[u8]]) -> Result<Vec<u8>, Error> {
183        Ok(self.mac(parts)?.finalize().into_bytes().to_vec())
184    }
185    fn verify(&self, parts: &[&[u8]], tag: &[u8]) -> Result<(), Error> {
186        self.mac(parts)?
187            .verify_slice(tag)
188            .map_err(|_| failure("authentication failed"))
189    }
190}
191
192/// Explicit coordinator endpoints and resource policy. Key bytes are read from
193/// the named local file and never included in reports, arguments or Debug values.
194#[derive(Clone, Debug)]
195pub struct Options {
196    /// Explicit trusted worker listeners; at least two distinct endpoints.
197    pub workers: Vec<SocketAddr>,
198    /// Local path to the shared 32-byte authentication secret.
199    pub key_file: PathBuf,
200    /// Per-network-operation and per-worker-unit bound.
201    pub timeout: Duration,
202    /// Retries of an uncommitted pure source after connection loss, at most eight.
203    pub retries: u8,
204}
205impl Options {
206    /// Two-or-more-worker policy with five-second units and two retries.
207    #[must_use]
208    pub fn new(workers: Vec<SocketAddr>, key_file: PathBuf) -> Self {
209        Self {
210            workers,
211            key_file,
212            timeout: Duration::from_secs(5),
213            retries: 2,
214        }
215    }
216    fn validate(&self) -> Result<(), Error> {
217        let distinct: HashSet<_> = self.workers.iter().collect();
218        if distinct.len() != self.workers.len()
219            || !(2..=64).contains(&distinct.len())
220            || self.timeout.is_zero()
221            || self.timeout > Duration::from_secs(60)
222            || self.retries > 8
223        {
224            return Err(failure(
225                "require 2..64 distinct workers, nonzero <=60s unit timeout and <=8 retries",
226            ));
227        }
228        Ok(())
229    }
230}
231
232/// Authenticated worker process facts, not remote attestation.
233#[derive(Clone, Debug, Serialize, Deserialize)]
234pub struct WorkerInfo {
235    /// Fresh independently initialized worker identity.
236    pub id: String,
237    /// Worker-reported operating-system process id.
238    pub process_id: u32,
239    /// Complete WORK frames sent to this authenticated instance, including retries.
240    pub sent_units: u64,
241    /// Complete authenticated RESULT frames received, before structural admission.
242    pub result_units: u64,
243}
244/// Actual work/retry facts; logical model counters include only ordered commit.
245#[derive(Clone, Debug, Default, Serialize, Deserialize)]
246pub struct Usage {
247    /// Requested trusted endpoints.
248    pub configured_workers: usize,
249    /// Independently authenticated process identities actually contacted.
250    pub workers: Vec<WorkerInfo>,
251    /// Source units assigned (excluding retries and startup pings).
252    pub assigned_units: u64,
253    /// Complete source results imported/admitted at ordered consumption.
254    pub completed_units: u64,
255    /// Actual connection-loss retries of uncommitted work.
256    pub retries: u64,
257    /// Largest completed source level/epoch.
258    pub epoch: u64,
259    /// Membership queries against the batch's real coordinator seen set.
260    pub membership_queries: u64,
261    /// Authenticated duplicate frames ignored without committing work twice.
262    pub duplicate_frames: u64,
263    /// Units left incomplete after retry policy, including unused later work.
264    pub failed_units: u64,
265    /// Required transport/admission failures reached in source order.
266    pub failures: Vec<String>,
267}
268
269#[derive(Clone, Copy)]
270struct Ticket {
271    epoch: u64,
272    work: u64,
273    attempt: u64,
274}
275#[derive(Clone, Copy)]
276struct Binding {
277    identity: [u8; 32],
278    run: [u8; 32],
279    client: [u8; 32],
280    server: [u8; 32],
281    ticket: Ticket,
282}
283impl Binding {
284    fn hello(&self) -> Vec<u8> {
285        let mut bytes = Vec::with_capacity(128);
286        bytes.extend_from_slice(MAGIC);
287        bytes.extend_from_slice(&self.identity);
288        bytes.extend_from_slice(&self.run);
289        bytes.extend_from_slice(&self.client);
290        for n in [self.ticket.epoch, self.ticket.work, self.ticket.attempt] {
291            bytes.extend_from_slice(&n.to_le_bytes());
292        }
293        bytes
294    }
295    fn context(&self) -> Vec<u8> {
296        let mut bytes = self.hello();
297        bytes.extend_from_slice(&self.server);
298        bytes
299    }
300}
301
302struct Channel {
303    stream: TcpStream,
304    binding: Binding,
305    send: u64,
306    receive: u64,
307    coordinator: bool,
308    deadline: Instant,
309}
310impl Channel {
311    fn direction(coordinator: bool) -> &'static [u8] {
312        if coordinator {
313            b"coordinator frame"
314        } else {
315            b"worker frame"
316        }
317    }
318    fn write(&mut self, key: &Secret, kind: u8, payload: &[u8]) -> Result<(), Error> {
319        if payload.len() > MAX_FRAME_BYTES {
320            return Err(failure("frame byte admission"));
321        }
322        let mut header = [0; 17];
323        header[0] = kind;
324        header[1..9].copy_from_slice(&self.send.to_le_bytes());
325        header[9..].copy_from_slice(&(payload.len() as u64).to_le_bytes());
326        let tag = key.tag(&[
327            Self::direction(self.coordinator),
328            &self.binding.context(),
329            &header,
330            payload,
331        ])?;
332        write_until(&mut self.stream, &header, self.deadline)?;
333        write_until(&mut self.stream, payload, self.deadline)?;
334        write_until(&mut self.stream, &tag, self.deadline)?;
335        self.send = self
336            .send
337            .checked_add(1)
338            .ok_or_else(|| failure("frame sequence exhausted"))?;
339        Ok(())
340    }
341    fn read(&mut self, key: &Secret, usage: &Mutex<Usage>) -> Result<(u8, Vec<u8>), Error> {
342        for _ in 0..64 {
343            let mut header = [0; 17];
344            read_until(&mut self.stream, &mut header, self.deadline)?;
345            let seq = u64::from_le_bytes(header[1..9].try_into().map_err(failure)?);
346            let len = usize::try_from(u64::from_le_bytes(header[9..].try_into().map_err(failure)?))
347                .map_err(failure)?;
348            if len > MAX_FRAME_BYTES {
349                return Err(failure("frame byte admission"));
350            }
351            let mut payload = Vec::new();
352            payload.try_reserve_exact(len).map_err(failure)?;
353            payload.resize(len, 0);
354            read_until(&mut self.stream, &mut payload, self.deadline)?;
355            let mut tag = [0; 32];
356            read_until(&mut self.stream, &mut tag, self.deadline)?;
357            key.verify(
358                &[
359                    Self::direction(!self.coordinator),
360                    &self.binding.context(),
361                    &header,
362                    &payload,
363                ],
364                &tag,
365            )?;
366            if seq < self.receive {
367                usage
368                    .lock()
369                    .unwrap_or_else(PoisonError::into_inner)
370                    .duplicate_frames += 1;
371                continue;
372            }
373            if seq != self.receive {
374                return Err(failure("future or stale session sequence"));
375            }
376            self.receive = self
377                .receive
378                .checked_add(1)
379                .ok_or_else(|| failure("frame sequence exhausted"))?;
380            return Ok((header[0], payload));
381        }
382        Err(failure("duplicate frame admission"))
383    }
384}
385
386fn timeouts(stream: &TcpStream, timeout: Duration) -> Result<(), Error> {
387    stream
388        .set_nodelay(true)
389        .and_then(|()| stream.set_read_timeout(Some(timeout)))
390        .and_then(|()| stream.set_write_timeout(Some(timeout)))
391        .map_err(io)
392}
393fn connect(
394    address: SocketAddr,
395    key: &Secret,
396    identity: [u8; 32],
397    run: [u8; 32],
398    ticket: Ticket,
399    timeout: Duration,
400) -> Result<(Channel, WorkerInfo), Error> {
401    let deadline = Instant::now() + timeout;
402    let mut stream = TcpStream::connect_timeout(&address, timeout).map_err(io)?;
403    timeouts(&stream, timeout)?;
404    let mut binding = Binding {
405        identity,
406        run,
407        client: random()?,
408        server: [0; 32],
409        ticket,
410    };
411    let hello = binding.hello();
412    let tag = key.tag(&[b"hello", &hello])?;
413    write_until(&mut stream, &hello, deadline)?;
414    write_until(&mut stream, &tag, deadline)?;
415    let mut ack = [0; 69];
416    let mut mac = [0; 32];
417    // A worker drops a session whose HELLO fails authentication without an
418    // authenticated reply, so name the likely causes of that silence.
419    read_until(&mut stream, &mut ack, deadline)
420        .and_then(|()| read_until(&mut stream, &mut mac, deadline))
421        .map_err(|error| {
422            io(format!(
423                "worker {address} closed or did not answer the authenticated handshake \
424                 (check that every role reads the same --worker-key file and that the \
425                 worker is running and not stalled; a different dive build answers \
426                 with an identity mismatch instead): {}",
427                error
428                    .message()
429                    .trim_start_matches("DIVE-S0502: distributed I/O: ")
430            ))
431        })?;
432    key.verify(&[b"ack", &hello, &ack], &mac)?;
433    if ack[0] != 0 {
434        return Err(failure(
435            "worker source/config/executable/policy/startup identity mismatch",
436        ));
437    }
438    binding.server.copy_from_slice(&ack[1..33]);
439    let info = WorkerInfo {
440        id: hex(&ack[33..65]),
441        process_id: u32::from_le_bytes(ack[65..].try_into().map_err(failure)?),
442        sent_units: 0,
443        result_units: 0,
444    };
445    Ok((
446        Channel {
447            stream,
448            binding,
449            send: 0,
450            receive: 0,
451            coordinator: true,
452            deadline,
453        },
454        info,
455    ))
456}
457fn accept(
458    mut stream: TcpStream,
459    key: &Secret,
460    identity: [u8; 32],
461    worker: [u8; 32],
462    timeout: Duration,
463) -> Result<Channel, Error> {
464    let deadline = Instant::now() + timeout;
465    // BSD-derived systems (macOS) let an accepted socket inherit O_NONBLOCK
466    // from the polled listener; Linux does not. The deadline-bounded reads
467    // below need a blocking socket, or a frame still in flight fails at once.
468    stream.set_nonblocking(false).map_err(io)?;
469    timeouts(&stream, timeout)?;
470    let mut hello = [0; 128];
471    let mut tag = [0; 32];
472    read_until(&mut stream, &mut hello, deadline)?;
473    read_until(&mut stream, &mut tag, deadline)?;
474    key.verify(&[b"hello", &hello], &tag)?;
475    if &hello[..8] != MAGIC {
476        return Err(failure("worker protocol version"));
477    }
478    if hello[8..40] != identity {
479        let mut rejection = [0; 69];
480        rejection[0] = 1;
481        let tag = key.tag(&[b"ack", &hello, &rejection])?;
482        write_until(&mut stream, &rejection, deadline)?;
483        write_until(&mut stream, &tag, deadline)?;
484        return Err(failure(
485            "source/config/executable/policy/startup identity mismatch",
486        ));
487    }
488    let word = |at| -> Result<u64, Error> {
489        Ok(u64::from_le_bytes(
490            hello[at..at + 8].try_into().map_err(failure)?,
491        ))
492    };
493    let mut binding = Binding {
494        identity,
495        run: hello[40..72].try_into().map_err(failure)?,
496        client: hello[72..104].try_into().map_err(failure)?,
497        server: random()?,
498        ticket: Ticket {
499            epoch: word(104)?,
500            work: word(112)?,
501            attempt: word(120)?,
502        },
503    };
504    let mut ack = [0; 69];
505    ack[1..33].copy_from_slice(&binding.server);
506    ack[33..65].copy_from_slice(&worker);
507    ack[65..].copy_from_slice(&std::process::id().to_le_bytes());
508    let tag = key.tag(&[b"ack", &hello, &ack])?;
509    write_until(&mut stream, &ack, deadline)?;
510    write_until(&mut stream, &tag, deadline)?;
511    // Keep only independently checked fixed-size identity/session fields.
512    binding.identity = identity;
513    Ok(Channel {
514        stream,
515        binding,
516        send: 0,
517        receive: 0,
518        coordinator: false,
519        deadline,
520    })
521}
522
523fn identity(base: &[u8], store: &Store) -> Result<([u8; 32], Vec<u8>), Error> {
524    store.strings().seal();
525    let image = dive_core::store::snapshot::capture(store, MAX_DAG_BYTES, &|| true)?;
526    let mut hash = Sha256::new();
527    hash.update(b"dive distributed pure core 0151 v1\0");
528    hash.update((base.len() as u64).to_le_bytes());
529    hash.update(base);
530    hash.update(&image);
531    Ok((hash.finalize().into(), image))
532}
533
534/// One coordinator's authenticated session and immutable endpoint policy.
535pub struct Coordinator {
536    options: Options,
537    key: Secret,
538    identity: [u8; 32],
539    run: [u8; 32],
540    startup: Startup,
541    usage: Mutex<Usage>,
542}
543
544// Bounded reply bytes plus the matching issuing authority. Import only when
545// this source reaches ordered consumption, preserving Store/history authority.
546pub(crate) struct Received {
547    exchange: Exchange,
548    payload: Result<Vec<u8>, Error>,
549    epoch: u64,
550}
551impl Coordinator {
552    /// Authenticates all endpoints against this compiled model before search.
553    ///
554    /// # Errors
555    /// Policy, identity, authentication, framing, startup admission or I/O failure.
556    pub fn new(options: Options, base_identity: &[u8], store: &Arc<Store>) -> Result<Self, Error> {
557        options.validate()?;
558        let key = Secret::read(&options.key_file)?;
559        let (identity, _) = identity(base_identity, store)?;
560        let run = random()?;
561        let startup = Startup::new(
562            &Heap::new(Arc::clone(store), true, false),
563            MAX_DAG_BYTES,
564            &|| true,
565        )?;
566        let owner = Self {
567            usage: Mutex::new(Usage {
568                configured_workers: options.workers.len(),
569                ..Usage::default()
570            }),
571            options,
572            key,
573            identity,
574            run,
575            startup,
576        };
577        for &address in &owner.options.workers {
578            let (mut channel, info) = connect(
579                address,
580                &owner.key,
581                identity,
582                run,
583                Ticket {
584                    epoch: 0,
585                    work: u64::MAX,
586                    attempt: 0,
587                },
588                owner.options.timeout,
589            )?;
590            channel.write(&owner.key, PING, &[])?;
591            if channel.read(&owner.key, &owner.usage)?.0 != PONG {
592                return Err(failure("startup acknowledgement"));
593            }
594            owner.worker(info);
595        }
596        if owner.usage().workers.len() < 2 {
597            return Err(failure("two independently initialized workers required"));
598        }
599        Ok(owner)
600    }
601    fn worker(&self, info: WorkerInfo) {
602        let mut usage = self.usage.lock().unwrap_or_else(PoisonError::into_inner);
603        if !usage.workers.iter().any(|w| w.id == info.id) {
604            usage.workers.push(info);
605        }
606    }
607    fn unit_fact(&self, id: &str, result: bool) -> Result<(), Error> {
608        let mut usage = self.usage.lock().unwrap_or_else(PoisonError::into_inner);
609        let info = usage
610            .workers
611            .iter_mut()
612            .find(|worker| worker.id == id)
613            .ok_or_else(|| failure("unregistered authenticated worker"))?;
614        let count = if result {
615            &mut info.result_units
616        } else {
617            &mut info.sent_units
618        };
619        *count = count
620            .checked_add(1)
621            .ok_or_else(|| failure("worker unit counter overflow"))?;
622        Ok(())
623    }
624    /// Actual contacted processes, work and retry facts.
625    #[must_use]
626    pub fn usage(&self) -> Usage {
627        self.usage
628            .lock()
629            .unwrap_or_else(PoisonError::into_inner)
630            .clone()
631    }
632    pub(crate) fn record_failure(&self, error: &Error) {
633        if error.message().starts_with("DIVE-S05") {
634            let mut facts = self.usage.lock().unwrap_or_else(PoisonError::into_inner);
635            if !facts
636                .failures
637                .iter()
638                .any(|failure| failure == error.message())
639            {
640                facts.failures.push(error.message().into());
641            }
642        }
643    }
644    pub(crate) fn width(&self) -> usize {
645        self.options.workers.len()
646    }
647
648    #[allow(
649        clippy::too_many_arguments,
650        reason = "one bounded batch borrows the actual committed seen authority"
651    )]
652    pub(crate) fn expand(
653        &self,
654        plan: &Plan,
655        store: &Arc<Store>,
656        rows: &[Handle],
657        seen: &crate::Seen,
658        epoch: u64,
659        poll: &(dyn Fn() -> bool + Sync),
660    ) -> Result<Vec<Option<Received>>, Error> {
661        let width = plan.vars.len();
662        if width == 0 || !rows.len().is_multiple_of(width) || rows.len() / width > self.width() {
663            return Err(failure("remote batch dimensions"));
664        }
665        let heap = Heap::new(Arc::clone(store), true, false);
666        let mut jobs = Vec::new();
667        jobs.try_reserve_exact(rows.len() / width)
668            .map_err(failure)?;
669        for row in rows.chunks_exact(width) {
670            jobs.push(Exchange::request_with_startup(
671                &heap,
672                row,
673                MAX_DAG_BYTES,
674                &|| !poll(),
675                &self.startup,
676            )?);
677        }
678        let first = {
679            let mut facts = self.usage.lock().unwrap_or_else(PoisonError::into_inner);
680            let first = facts.assigned_units;
681            facts.assigned_units = facts
682                .assigned_units
683                .checked_add(jobs.len() as u64)
684                .ok_or_else(|| failure("work ids exhausted"))?;
685            first
686        };
687        let received = std::thread::scope(|scope| {
688            let handles: Vec<_> = jobs
689                .into_iter()
690                .enumerate()
691                .map(|(i, (bytes, exchange))| {
692                    std::thread::Builder::new()
693                        .name("dive-network".into())
694                        .spawn_scoped(scope, move || {
695                            (
696                                exchange,
697                                self.job(first + i as u64, epoch, &bytes, seen, store, poll),
698                            )
699                        })
700                        .map_err(|error| failure(format!("network thread start: {error}")))
701                })
702                .collect::<Result<Vec<_>, _>>()?;
703            handles
704                .into_iter()
705                .map(|h| {
706                    h.join()
707                        .map_err(|_| failure("network worker thread failed"))
708                })
709                .collect::<Result<Vec<_>, _>>()
710        })?;
711        let mut result = Vec::new();
712        result.try_reserve_exact(received.len()).map_err(failure)?;
713        for (exchange, payload) in received {
714            result.push(Some(Received {
715                exchange,
716                payload,
717                epoch,
718            }));
719        }
720        Ok(result)
721    }
722
723    pub(crate) fn import(
724        &self,
725        received: Received,
726        plan: &Plan,
727        heap: &mut Heap,
728        poll: &(dyn Fn() -> bool + Sync),
729        metrics: &mut crate::Metrics,
730    ) -> Result<crate::parallel::Expanded, Error> {
731        let payload = received.payload?;
732        let (meta, dag) = wire::split(&payload)?;
733        let roots = received
734            .exchange
735            .receive_reply(heap, dag, MAX_DAG_BYTES, &|| !poll())?;
736        let (expanded, counts) = wire::decode(
737            meta,
738            &roots,
739            plan,
740            usize::try_from(MAX_SUCCESSORS).map_err(failure)?,
741        )?;
742        counts.apply(metrics)?;
743        let mut facts = self.usage.lock().unwrap_or_else(PoisonError::into_inner);
744        facts.completed_units += 1;
745        facts.epoch = facts.epoch.max(received.epoch);
746        Ok(expanded)
747    }
748
749    fn job(
750        &self,
751        work: u64,
752        epoch: u64,
753        bytes: &[u8],
754        seen: &crate::Seen,
755        store: &Arc<Store>,
756        poll: &(dyn Fn() -> bool + Sync),
757    ) -> Result<Vec<u8>, Error> {
758        let mut final_error = failure("work was not attempted");
759        for attempt in 0..=self.options.retries {
760            if poll() {
761                return Err(Error::stop());
762            }
763            if attempt > 0 {
764                self.usage
765                    .lock()
766                    .unwrap_or_else(PoisonError::into_inner)
767                    .retries += 1;
768            }
769            let index = usize::try_from(work % (self.width() as u64)).map_err(failure)?;
770            let address = self.options.workers[(index + usize::from(attempt)) % self.width()];
771            let result = (|| {
772                let (mut channel, info) = connect(
773                    address,
774                    &self.key,
775                    self.identity,
776                    self.run,
777                    Ticket {
778                        epoch,
779                        work,
780                        attempt: u64::from(attempt),
781                    },
782                    self.options.timeout,
783                )?;
784                let worker_id = info.id.clone();
785                self.worker(info);
786                channel.write(&self.key, WORK, bytes)?;
787                self.unit_fact(&worker_id, false)?;
788                let heap = Heap::new(Arc::clone(store), true, false);
789                loop {
790                    if poll() {
791                        return Err(Error::stop());
792                    }
793                    let (kind, payload) = channel.read(&self.key, &self.usage)?;
794                    match kind {
795                        QUERY => {
796                            if payload.len() < 8 || payload.len() > 8 + KEY_BYTES {
797                                return Err(failure("membership query admission"));
798                            }
799                            let key = u64::from_le_bytes(payload[..8].try_into().map_err(failure)?);
800                            let present = if let Some(exact) = &seen.exact {
801                                exact.contains_encoded(
802                                    key,
803                                    &payload[8..],
804                                    &heap,
805                                    KEY_BYTES,
806                                    poll,
807                                )?
808                            } else {
809                                seen.contains(key)
810                            };
811                            self.usage
812                                .lock()
813                                .unwrap_or_else(PoisonError::into_inner)
814                                .membership_queries += 1;
815                            channel.write(&self.key, ANSWER, &[u8::from(present)])?;
816                        }
817                        RESULT => {
818                            self.unit_fact(&worker_id, true)?;
819                            return Ok(payload);
820                        }
821                        FAILED => {
822                            return Err(failure(format!(
823                                "worker refused incomplete unit: {}",
824                                String::from_utf8_lossy(&payload)
825                            )));
826                        }
827                        _ => return Err(failure("unexpected worker message")),
828                    }
829                }
830            })();
831            match result {
832                Ok(bytes) => return Ok(bytes),
833                Err(error) if error.kind() == dive_core::ErrorKind::Stop => return Err(error),
834                Err(error) => {
835                    let retry = error.message().starts_with("DIVE-S0502:");
836                    final_error = error;
837                    if !retry {
838                        break;
839                    }
840                }
841            }
842        }
843        self.usage
844            .lock()
845            .unwrap_or_else(PoisonError::into_inner)
846            .failed_units += 1;
847        Err(final_error)
848    }
849}
850
851/// Independently prepared native worker. Its plan is fixed before any session.
852pub struct Worker {
853    plan: Plan,
854    store: Arc<Store>,
855    startup: Vec<u8>,
856    opts: Opts,
857    key: Secret,
858    identity: [u8; 32],
859    id: [u8; 32],
860    timeout: Duration,
861}
862impl Worker {
863    /// Binds the compiled startup and authentication secret for pure work.
864    ///
865    /// # Errors
866    /// Effects, quotient modes, private startup admission, keys or identity hashing.
867    pub fn new(
868        plan: Plan,
869        store: Arc<Store>,
870        opts: Opts,
871        base_identity: &[u8],
872        key_file: &Path,
873        timeout: Duration,
874    ) -> Result<Self, Error> {
875        if plan.effects.any || plan.view.is_some() || plan.symmetry.is_some() {
876            return Err(Error::unsupported("distributed effects, VIEW or symmetry"));
877        }
878        if timeout.is_zero() || timeout > Duration::from_secs(60) {
879            return Err(failure("worker timeout must be nonzero and <=60s"));
880        }
881        let (identity, startup) = identity(base_identity, &store)?;
882        Ok(Self {
883            plan,
884            store,
885            startup,
886            opts,
887            key: Secret::read(key_file)?,
888            identity,
889            id: random()?,
890            timeout,
891        })
892    }
893    /// Serves bounded authenticated units from a supplied listener until stopped.
894    /// `max_units` permits finite process lifetimes without a remote shutdown grant.
895    ///
896    /// # Errors
897    /// Listener ownership or setup failure. Bad sessions are rejected and counted.
898    pub fn serve(
899        &self,
900        listener: &TcpListener,
901        max_units: Option<u64>,
902        stop: &(dyn Fn() -> bool + Sync),
903    ) -> Result<Usage, Error> {
904        listener.set_nonblocking(true).map_err(io)?;
905        let usage = Mutex::new(Usage::default());
906        // The coordinator opens one session per source unit, so the next one
907        // usually arrives within a millisecond of the last. Poll quickly after
908        // a session and back off to the idle interval while nothing arrives.
909        let mut idle = AcceptBackoff::default();
910        while !stop()
911            && max_units.is_none_or(|n| {
912                usage
913                    .lock()
914                    .unwrap_or_else(PoisonError::into_inner)
915                    .completed_units
916                    < n
917            })
918        {
919            let stream = match listener.accept() {
920                Ok((s, _)) => {
921                    idle.reset();
922                    s
923                }
924                Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
925                    std::thread::sleep(idle.next_wait());
926                    continue;
927                }
928                Err(e) => return Err(io(e)),
929            };
930            let result = accept(stream, &self.key, self.identity, self.id, self.timeout).and_then(
931                |mut channel| {
932                    let result = self.unit(&mut channel, &usage);
933                    if let Err(error) = &result {
934                        let _ = channel.write(&self.key, FAILED, error.message().as_bytes());
935                    }
936                    result
937                },
938            );
939            if let Err(error) = result {
940                let mut facts = usage.lock().unwrap_or_else(PoisonError::into_inner);
941                if facts.failures.len() < 64 {
942                    facts.failures.push(error.message().into());
943                }
944            }
945        }
946        Ok(usage.into_inner().unwrap_or_else(PoisonError::into_inner))
947    }
948    fn unit(&self, channel: &mut Channel, usage: &Mutex<Usage>) -> Result<(), Error> {
949        let (kind, payload) = channel.read(&self.key, usage)?;
950        if kind == PING && payload.is_empty() {
951            return channel.write(&self.key, PONG, &[]);
952        }
953        if kind != WORK {
954            return Err(failure("work kind"));
955        }
956        let deadline = channel.deadline;
957        let poll = || Instant::now() >= deadline;
958        let work = (|| {
959            let store = dive_core::store::snapshot::restore(
960                &self.store,
961                &self.startup,
962                &self.startup,
963                MAX_DAG_BYTES,
964                &|| !poll(),
965            )?;
966            let mut machine = Machine::new(
967                store,
968                self.plan.vars.len(),
969                self.opts,
970                dive_core::memo::Memo::new(
971                    &self
972                        .plan
973                        .code
974                        .sites
975                        .iter()
976                        .map(dive_core::plan::MemoSite::key_len)
977                        .collect::<Vec<_>>(),
978                ),
979            );
980            machine.set_work_control(Some(Arc::new(Deadline(deadline))));
981            let startup = Startup::new(&machine.heap, MAX_DAG_BYTES, &|| !poll())?;
982            let (source, exchange) = Exchange::receive_request_with_startup(
983                &mut machine.heap,
984                &payload,
985                MAX_DAG_BYTES,
986                &|| !poll(),
987                &startup,
988            )?;
989            if source.len() != self.plan.vars.len() || source.iter().any(|h| h.is_empty()) {
990                return Err(failure("source row width/completeness"));
991            }
992            let mut member = RemoteMember {
993                channel,
994                key: &self.key,
995                usage,
996            };
997            let mut metrics = crate::Metrics::default();
998            let expanded = crate::parallel::expand_remote(
999                &mut machine,
1000                &self.plan,
1001                &source,
1002                MAX_SUCCESSORS,
1003                &poll,
1004                &mut metrics,
1005                &mut member,
1006            );
1007            let (meta, roots) = wire::encode(&expanded, &metrics, MAX_FRAME_BYTES)?;
1008            let dag = exchange.reply(&machine.heap, &roots, MAX_DAG_BYTES, &|| !poll())?;
1009            wire::join(&meta, &dag, MAX_FRAME_BYTES)
1010        })();
1011        match work {
1012            Ok(bytes) => {
1013                channel.write(&self.key, RESULT, &bytes)?;
1014                usage
1015                    .lock()
1016                    .unwrap_or_else(PoisonError::into_inner)
1017                    .completed_units += 1;
1018                Ok(())
1019            }
1020            Err(error) => {
1021                let mut facts = usage.lock().unwrap_or_else(PoisonError::into_inner);
1022                if facts.failures.len() < 64 {
1023                    facts.failures.push(error.message().into());
1024                }
1025                drop(facts);
1026                // The unit deadline may already have passed; give the bounded
1027                // FAILED report its own deadline so the coordinator learns why.
1028                channel.deadline = Instant::now() + self.timeout;
1029                let message = if Instant::now() >= deadline {
1030                    format!(
1031                        "the unit did not finish within the worker's --worker-timeout-ms of {} ms",
1032                        self.timeout.as_millis()
1033                    )
1034                } else {
1035                    error.message().to_owned()
1036                };
1037                channel.write(&self.key, FAILED, message.as_bytes())
1038            }
1039        }
1040    }
1041}
1042
1043#[derive(Debug)]
1044struct Deadline(Instant);
1045impl dive_core::eval::WorkControl for Deadline {
1046    fn proceed(&self) -> bool {
1047        Instant::now() < self.0
1048    }
1049}
1050
1051struct RemoteMember<'a> {
1052    channel: &'a mut Channel,
1053    key: &'a Secret,
1054    usage: &'a Mutex<Usage>,
1055}
1056impl crate::parallel::Membership for RemoteMember<'_> {
1057    fn contains(&mut self, key: u64, identity: &[Handle], heap: &Heap) -> Result<bool, Error> {
1058        let bytes = dive_core::key::canonical(heap, identity, KEY_BYTES, &|| {
1059            Instant::now() < self.channel.deadline
1060        })
1061        .map_err(|e| failure(format!("canonical worker key: {e:?}")))?;
1062        let mut query = Vec::new();
1063        query.extend_from_slice(&key.to_le_bytes());
1064        query.extend_from_slice(&bytes);
1065        self.channel.write(self.key, QUERY, &query)?;
1066        let (kind, answer) = self.channel.read(self.key, self.usage)?;
1067        if kind != ANSWER {
1068            return Err(failure("membership answer kind"));
1069        }
1070        match answer.as_slice() {
1071            [0] => Ok(false),
1072            [1] => Ok(true),
1073            _ => Err(failure("membership answer")),
1074        }
1075    }
1076}
1077
1078#[cfg(test)]
1079mod fault_tests;
1080
1081#[cfg(test)]
1082mod tests {
1083    #![allow(
1084        clippy::expect_used,
1085        reason = "local authenticated worker backend controls"
1086    )]
1087    use super::*;
1088    use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
1089
1090    struct Stop(Arc<AtomicBool>);
1091    impl Drop for Stop {
1092        fn drop(&mut self) {
1093            self.0.store(true, Ordering::Relaxed);
1094        }
1095    }
1096
1097    #[test]
1098    fn accept_backoff_doubles_to_the_cap_and_resets_after_a_session() {
1099        let mut backoff = AcceptBackoff::default();
1100        let waits: Vec<Duration> = (0..12).map(|_| backoff.next_wait()).collect();
1101        assert_eq!(waits[0], ACCEPT_POLL_MIN);
1102        assert_eq!(waits[1], ACCEPT_POLL_MIN * 2);
1103        assert!(waits.windows(2).all(|w| w[1] >= w[0]));
1104        assert!(waits.iter().all(|w| *w <= ACCEPT_POLL_MAX));
1105        // 50 us doubled eight times exceeds 10 ms, so the cap holds from there on.
1106        assert_eq!(waits[8], ACCEPT_POLL_MAX);
1107        assert_eq!(waits[11], ACCEPT_POLL_MAX);
1108        backoff.reset();
1109        assert_eq!(backoff.next_wait(), ACCEPT_POLL_MIN);
1110    }
1111
1112    // Issue #1022: on macOS an accepted socket inherits the polled listener's
1113    // O_NONBLOCK, so a HELLO not yet in flight failed at once as a timeout.
1114    #[test]
1115    fn accepted_session_waits_for_a_late_hello_from_a_nonblocking_listener() {
1116        let listener = TcpListener::bind("127.0.0.1:0").expect("local listener");
1117        listener
1118            .set_nonblocking(true)
1119            .expect("nonblocking listener");
1120        let address = listener.local_addr().expect("address");
1121        let client = std::thread::spawn(move || {
1122            let mut stream = TcpStream::connect(address).expect("connect");
1123            std::thread::sleep(Duration::from_millis(100));
1124            stream.write_all(&[0; 160]).expect("late hello");
1125            stream
1126        });
1127        let stream = loop {
1128            match listener.accept() {
1129                Ok((stream, _)) => break stream,
1130                Err(e) => {
1131                    assert_eq!(e.kind(), std::io::ErrorKind::WouldBlock, "accept: {e}");
1132                    std::thread::sleep(Duration::from_millis(1));
1133                }
1134            }
1135        };
1136        let error = accept(
1137            stream,
1138            &Secret([37; 32]),
1139            [0; 32],
1140            [0; 32],
1141            Duration::from_secs(5),
1142        )
1143        .err()
1144        .expect("an unauthenticated hello is rejected");
1145        // The whole zero HELLO and tag are read and fail authentication, not I/O.
1146        assert_eq!(
1147            error.message(),
1148            "DIVE-S0500: distributed safety: authentication failed"
1149        );
1150        drop(client.join().expect("client"));
1151    }
1152
1153    #[test]
1154    fn two_independent_native_workers_complete_actual_ordered_search() {
1155        static SERIAL: AtomicU64 = AtomicU64::new(0);
1156        for exact in [false, true] {
1157            let path = std::env::temp_dir().join(format!(
1158                "dive-worker-key-{}-{}",
1159                std::process::id(),
1160                SERIAL.fetch_add(1, Ordering::Relaxed)
1161            ));
1162            std::fs::write(&path, [37; 32]).expect("test key");
1163            let store = Arc::new(Store::new(true));
1164            let plan = crate::tests::frontier_plan(&store, false).expect("coordinator plan");
1165            let local = crate::explore_with_storage(
1166                &plan,
1167                &store,
1168                crate::Config {
1169                    opts: Opts::default(),
1170                    max_states: None,
1171                    max_depth: None,
1172                    workers: 1,
1173                    warn: true,
1174                    sample: None,
1175                },
1176                &|| false,
1177                None,
1178                None,
1179                &crate::storage::Storage {
1180                    exact,
1181                    ..crate::storage::Storage::default()
1182                },
1183            );
1184            let mut workers = Vec::new();
1185            let mut listeners = Vec::new();
1186            let mut endpoints = Vec::new();
1187            for _ in 0..2 {
1188                let listener = TcpListener::bind("127.0.0.1:0").expect("local listener");
1189                endpoints.push(listener.local_addr().expect("address"));
1190                let owner = Arc::new(Store::new(true));
1191                let plan = crate::tests::frontier_plan(&owner, false).expect("independent plan");
1192                workers.push(
1193                    Worker::new(
1194                        plan,
1195                        owner,
1196                        Opts::default(),
1197                        b"same independently compiled test",
1198                        &path,
1199                        Duration::from_secs(5),
1200                    )
1201                    .expect("worker"),
1202                );
1203                listeners.push(listener);
1204            }
1205            let stop = Arc::new(AtomicBool::new(false));
1206            std::thread::scope(|scope| {
1207                let _guard = Stop(Arc::clone(&stop));
1208                for (worker, listener) in workers.iter().zip(&listeners) {
1209                    let stop = &stop;
1210                    scope.spawn(move || {
1211                        worker
1212                            .serve(listener, None, &|| stop.load(Ordering::Relaxed))
1213                            .expect("serve")
1214                    });
1215                }
1216                let coordinator = Coordinator::new(
1217                    Options::new(endpoints, path.clone()),
1218                    b"same independently compiled test",
1219                    &store,
1220                )
1221                .expect("authenticated coordinator");
1222                let out = crate::explore_distributed(
1223                    &plan,
1224                    &store,
1225                    crate::Config {
1226                        opts: Opts::default(),
1227                        max_states: None,
1228                        max_depth: None,
1229                        workers: 1,
1230                        warn: true,
1231                        sample: None,
1232                    },
1233                    &|| false,
1234                    None,
1235                    &crate::storage::Storage {
1236                        exact,
1237                        ..crate::storage::Storage::default()
1238                    },
1239                    &coordinator,
1240                );
1241                assert_eq!(out.verdict, local.verdict, "{:?}", out.verdict);
1242                assert_eq!(out.stats, local.stats);
1243                let facts = coordinator.usage();
1244                assert_eq!(facts.workers.len(), 2);
1245                assert_eq!(facts.completed_units, 128);
1246                assert_eq!(facts.assigned_units, 128);
1247                assert_eq!(facts.membership_queries, 128);
1248                assert_eq!(facts.failures, [] as [String; 0]);
1249            });
1250            std::fs::remove_file(path).expect("remove key");
1251        }
1252    }
1253
1254    #[test]
1255    fn authentication_binds_identity_run_epoch_work_attempt_and_payload() {
1256        let key = Secret([17; 32]);
1257        let binding = Binding {
1258            identity: [1; 32],
1259            run: [2; 32],
1260            client: [3; 32],
1261            server: [4; 32],
1262            ticket: Ticket {
1263                epoch: 5,
1264                work: 6,
1265                attempt: 7,
1266            },
1267        };
1268        let context = binding.context();
1269        let tag = key
1270            .tag(&[&context, b"work", b"actual payload"])
1271            .expect("tag");
1272        assert!(
1273            key.verify(&[&context, b"work", b"actual payload"], &tag)
1274                .is_ok()
1275        );
1276        for at in [8, 40, 72, 104, 112, 120, 128] {
1277            let mut altered = context.clone();
1278            altered[at] ^= 1;
1279            assert!(
1280                key.verify(&[&altered, b"work", b"actual payload"], &tag)
1281                    .is_err()
1282            );
1283        }
1284        assert!(
1285            key.verify(&[&context, b"result", b"actual payload"], &tag)
1286                .is_err()
1287        );
1288        assert!(
1289            key.verify(&[&context, b"work", b"altered payload"], &tag)
1290                .is_err()
1291        );
1292    }
1293}