indicatrix-worker 0.7.2

Coordinator and render worker for an indicatrix-backed design catalogue. `serve` (default build) is the coordinator: it serves indicatrix-net's read-only design-library protocol to viewers over mutual TLS and accepts joining render workers on a separate worker port. The optional `worker` feature adds render capacity -- `join` (dial a coordinator and render its requests), `serve --render` (render with the coordinator's own CPU/GPU) and `render` (a one-shot scene-to-PNG CLI).
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
//! The viewer's own share of a `FinalImageRequest` (v16): [`ContributionSlot`] is the
//! rendezvous between the connection thread (which reads the `CONTRIBUTION` off the
//! wire, see [`ContributionSlot::receive`]) and the job's producer thread (which waits
//! for it once its own lanes finish, see [`ContributionSlot::await_until`]).
//!
//! # Why a rendezvous, not a channel
//!
//! Both directions need more than "hand over a value": the producer thread must be able
//! to give up after a bounded wait (reclaiming the range itself) while still never
//! reclaiming mid-upload, and the connection thread must know whether a contribution
//! that finishes arriving late is still wanted at all. A `Mutex<SlotState>` plus
//! `Condvar` gives both sides that visibility into the other's progress; a plain
//! one-shot channel would not.

use super::TimeoutCache;
use crate::stream_emit::TimeoutRead;
use glam::Vec3;
use indicatrix_dispatch::SampleRange;
use indicatrix_net::messages::{
    ContributionError, ContributionHeader, ExpectedContribution, NetError,
};
use std::{
    io::Read,
    sync::{Condvar, Mutex, PoisonError},
    time::{Duration, Instant},
};

/// The default wait for a reserved viewer contribution once the server's own lanes
/// finish, when a request doesn't override it -- see [`super::super::coordinator::job::JobConfig::contribution_wait`].
pub const DEFAULT_CONTRIBUTION_WAIT: Duration = Duration::from_secs(30);

/// Per-read timeout while a `CONTRIBUTION` payload streams in -- mirrors
/// `crate::assets::fetch`'s own asset-payload read timeout.
pub const CONTRIBUTION_PAYLOAD_READ_TIMEOUT: Duration = Duration::from_secs(30);

/// What [`ContributionSlot::await_until`] returned.
pub enum Awaited {
    /// The contribution arrived, was valid, and is ready to fold in.
    Arrived(Vec<Vec3>),
    /// No usable contribution: the wait elapsed, or what arrived was invalid. `reason`
    /// is logged by the caller, never sent to the viewer (see this crate's coordinator
    /// job doc comment on why a reclaim is silent).
    Reclaim {
        /// Human-readable reason, for a `tracing::info!` at the call site.
        reason: String,
    },
    /// The request was cancelled while waiting.
    Cancelled,
}

/// The slot's internal state, guarded by [`ContributionSlot::state`].
enum SlotState {
    /// Nothing has arrived yet.
    Waiting,
    /// The connection thread is mid-read of the payload frame -- never reclaimed while
    /// in this state (see [`ContributionSlot::await_until`]).
    Receiving,
    /// A valid contribution, ready to be taken exactly once.
    Arrived(Vec<Vec3>),
    /// What arrived did not decode, or didn't match the expected request/range/size.
    Invalid(String),
    /// [`ContributionSlot::await_until`] gave up (timeout, or an invalid arrival) and
    /// the server is tracing this range itself; a contribution that arrives after this
    /// is dropped (see [`ContributionSlot::receive`]).
    Reclaimed,
    /// [`ContributionSlot::await_until`] already returned [`Awaited::Arrived`]; the
    /// slot is spent.
    Taken,
}

/// The rendezvous for one `FinalImageRequest`'s reserved viewer range -- see the module
/// doc comment.
pub struct ContributionSlot {
    reserved: SampleRange,
    width: u32,
    height: u32,
    state: Mutex<SlotState>,
    changed: Condvar,
}

impl std::fmt::Debug for ContributionSlot {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("ContributionSlot")
            .field("reserved", &self.reserved)
            .field("width", &self.width)
            .field("height", &self.height)
            .finish_non_exhaustive()
    }
}

impl ContributionSlot {
    /// A fresh slot for `reserved` (the viewer's tail of a `width x height` picture),
    /// nothing arrived yet.
    #[must_use]
    pub const fn new(reserved: SampleRange, width: u32, height: u32) -> Self {
        Self {
            reserved,
            width,
            height,
            state: Mutex::new(SlotState::Waiting),
            changed: Condvar::new(),
        }
    }

    /// The reserved range this slot answers for.
    #[must_use]
    pub const fn reserved(&self) -> SampleRange {
        self.reserved
    }

    /// The connection thread's side: `header` has already been read off the wire (its
    /// `ClientMessage::Contribution` envelope), so this reads and validates the
    /// payload frame that follows, bounded exactly like `messages::contribution`
    /// documents.
    ///
    /// Marks the slot `Receiving` before reading (so [`Self::await_until`] never
    /// reclaims mid-upload), then `Arrived`/`Invalid` once the read finishes -- unless
    /// the slot was already `Reclaimed`/`Taken` by then, in which case the bytes are
    /// simply dropped with an info log (the producer thread has moved on).
    ///
    /// # Errors
    ///
    /// [`NetError`] only for [`ContributionError::TooLarge`]/[`ContributionError::Framing`]
    /// (the connection is out of sync and must be dropped); every other outcome
    /// (including an invalid contribution) is `Ok(())`.
    pub(crate) fn receive<S: Read + TimeoutRead>(
        &self,
        stream: &mut S,
        header: &ContributionHeader,
        request_id: u32,
        timeouts: &mut TimeoutCache,
    ) -> Result<(), NetError> {
        // A slot the producer thread has already moved past (Reclaimed/Taken) stays
        // exactly there -- overwriting it with `Receiving` would erase the very fact
        // `finish_receive` needs to recognise this arrival as late.
        if !self.already_spent() {
            self.set_state(SlotState::Receiving);
        }
        timeouts
            .apply(stream, Some(CONTRIBUTION_PAYLOAD_READ_TIMEOUT))
            .map_err(|e| NetError::Framing(indicatrix_net::framing::FramingError::Io(e)))?;

        let expect = ExpectedContribution {
            request_id,
            first_sample: self.reserved.first_sample,
            samples: self.reserved.samples,
            width: self.width,
            height: self.height,
        };
        let mut decoder = indicatrix_net::radiance::PayloadDecoder::new();
        let outcome = indicatrix_net::messages::read_contribution_payload(
            stream,
            header,
            expect,
            &mut decoder,
        );
        self.finish_receive(request_id, outcome)
    }

    /// Sets `state` and wakes [`Self::await_until`].
    fn set_state(&self, state: SlotState) {
        *self.state.lock().unwrap_or_else(PoisonError::into_inner) = state;
        self.changed.notify_all();
    }

    /// The second half of [`Self::receive`]: turns the decode outcome into the slot's
    /// next state (or a hard [`NetError`]), split out so `receive` stays short.
    fn finish_receive(
        &self,
        request_id: u32,
        outcome: Result<Vec<Vec3>, ContributionError>,
    ) -> Result<(), NetError> {
        match outcome {
            Ok(sum) => {
                if self.already_spent() {
                    tracing::info!(
                        request_id,
                        "a CONTRIBUTION arrived after its range was already reclaimed or taken; dropping it"
                    );
                } else {
                    self.set_state(SlotState::Arrived(sum));
                }
                Ok(())
            }
            Err(ContributionError::TooLarge {
                payload_len,
                raw_len,
            }) => Err(NetError::Framing(
                indicatrix_net::framing::FramingError::FrameTooLarge {
                    len: payload_len,
                    max: raw_len,
                },
            )),
            Err(ContributionError::Framing(e)) => Err(NetError::Framing(e)),
            Err(e @ (ContributionError::Mismatch(_) | ContributionError::Radiance(_))) => {
                let reason = e.to_string();
                if self.already_spent() {
                    tracing::info!(
                        request_id,
                        "an invalid CONTRIBUTION arrived after its range was already reclaimed or \
                         taken; dropping it: {reason}"
                    );
                } else {
                    self.set_state(SlotState::Invalid(reason));
                }
                Ok(())
            }
        }
    }

    /// Whether [`Self::await_until`] has already moved past this slot (`Reclaimed`/`Taken`).
    fn already_spent(&self) -> bool {
        matches!(
            *self.state.lock().unwrap_or_else(PoisonError::into_inner),
            SlotState::Reclaimed | SlotState::Taken
        )
    }

    /// The producer thread's side: waits up to `wait` (extended for as long as a
    /// receive is actually in flight) in 50 ms slices for the contribution to arrive,
    /// or `cancelled` to report the request has ended.
    ///
    /// Returns [`Awaited::Arrived`] immediately once the slot holds one (marking it
    /// `Taken`); [`Awaited::Reclaim`] immediately once it holds an invalid one (marking
    /// it `Reclaimed`, no wait), or once `wait` elapses with nothing usable (also
    /// `Reclaimed`); [`Awaited::Cancelled`] as soon as `cancelled()` says so.
    pub fn await_until(&self, wait: Duration, cancelled: impl Fn() -> bool) -> Awaited {
        let mut deadline = Instant::now() + wait;
        let mut guard = self.state.lock().unwrap_or_else(PoisonError::into_inner);
        loop {
            match &*guard {
                SlotState::Arrived(_) => {
                    let SlotState::Arrived(sum) = std::mem::replace(&mut *guard, SlotState::Taken)
                    else {
                        unreachable!("just matched SlotState::Arrived");
                    };
                    return Awaited::Arrived(sum);
                }
                SlotState::Invalid(reason) => {
                    let reason = reason.clone();
                    *guard = SlotState::Reclaimed;
                    return Awaited::Reclaim { reason };
                }
                // An upload is in flight: never reclaim mid-read -- keep pushing the
                // deadline out for as long as it continues.
                SlotState::Receiving => deadline = Instant::now() + wait,
                SlotState::Waiting | SlotState::Reclaimed | SlotState::Taken => {}
            }
            if cancelled() {
                return Awaited::Cancelled;
            }
            if Instant::now() >= deadline {
                *guard = SlotState::Reclaimed;
                return Awaited::Reclaim {
                    reason: format!("no contribution within {wait:?}"),
                };
            }
            guard = self
                .changed
                .wait_timeout(guard, Duration::from_millis(50))
                .unwrap_or_else(PoisonError::into_inner)
                .0;
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use indicatrix_net::{messages::PayloadEncoding, radiance::PayloadEncoder};
    use std::{
        io::Cursor,
        sync::{Arc, mpsc},
        thread,
    };

    /// A `Read + TimeoutRead` double over an in-memory buffer -- timeouts are a no-op,
    /// exactly enough for these single-threaded/scoped-thread tests.
    struct MockStream(Cursor<Vec<u8>>);

    impl Read for MockStream {
        fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
            self.0.read(buf)
        }
    }

    impl TimeoutRead for MockStream {
        fn set_read_timeout(&mut self, _duration: Option<Duration>) -> std::io::Result<()> {
            Ok(())
        }
    }

    fn contribution_wire(
        request_id: u32,
        range: SampleRange,
        width: u32,
        height: u32,
        value: f32,
    ) -> Vec<u8> {
        let sum = vec![Vec3::splat(value); (width * height) as usize];
        let mut encoder = PayloadEncoder::new(PayloadEncoding::Raw);
        let mut buf = Vec::new();
        indicatrix_net::messages::write_contribution_message(
            &mut buf,
            request_id,
            (range.first_sample, range.samples),
            width,
            height,
            &sum,
            &mut encoder,
        )
        .unwrap();
        buf
    }

    fn header_and_stream(wire: Vec<u8>) -> (ContributionHeader, MockStream) {
        let mut cursor = Cursor::new(wire);
        let msg: indicatrix_net::messages::ClientMessage =
            indicatrix_net::messages::read_message(&mut cursor).unwrap();
        let indicatrix_net::messages::ClientMessage::Contribution(header) = msg else {
            panic!("expected ClientMessage::Contribution");
        };
        (header, MockStream(cursor))
    }

    /// An arrival already sitting in the slot is returned immediately, without waiting.
    #[test]
    fn an_arrival_before_the_wait_is_returned_immediately() {
        let range = SampleRange::new(10, 4);
        let slot = ContributionSlot::new(range, 2, 2);
        let wire = contribution_wire(7, range, 2, 2, 3.0);
        let (header, mut stream) = header_and_stream(wire);
        slot.receive(&mut stream, &header, 7, &mut TimeoutCache::new())
            .unwrap();

        let start = Instant::now();
        let Awaited::Arrived(sum) = slot.await_until(Duration::from_secs(30), || false) else {
            panic!("expected Arrived");
        };
        assert_eq!(sum, vec![Vec3::splat(3.0); 4]);
        assert!(start.elapsed() < Duration::from_secs(1));
    }

    /// Nothing ever arrives: the wait elapses and the slot reclaims. A contribution
    /// that shows up afterward is silently dropped (proven by `receive` still
    /// returning `Ok(())` and not panicking on an already-Reclaimed slot).
    #[test]
    fn a_timeout_reclaims_and_a_late_arrival_is_dropped() {
        let range = SampleRange::new(0, 2);
        let slot = ContributionSlot::new(range, 1, 1);
        let outcome = slot.await_until(Duration::from_millis(80), || false);
        assert!(matches!(outcome, Awaited::Reclaim { .. }));

        let wire = contribution_wire(1, range, 1, 1, 9.0);
        let (header, mut stream) = header_and_stream(wire);
        slot.receive(&mut stream, &header, 1, &mut TimeoutCache::new())
            .unwrap();
        // Still reclaimed -- the late arrival must not resurrect it.
        assert!(matches!(*slot.state.lock().unwrap(), SlotState::Reclaimed));
    }

    /// A contribution with the wrong range reclaims immediately, no wait needed.
    #[test]
    fn an_invalid_contribution_reclaims_without_waiting() {
        let range = SampleRange::new(0, 2);
        let slot = ContributionSlot::new(range, 1, 1);
        // Built for a different range than the slot expects.
        let wire = contribution_wire(1, SampleRange::new(5, 2), 1, 1, 1.0);
        let (header, mut stream) = header_and_stream(wire);
        slot.receive(&mut stream, &header, 1, &mut TimeoutCache::new())
            .unwrap();

        let start = Instant::now();
        let outcome = slot.await_until(Duration::from_secs(30), || false);
        assert!(matches!(outcome, Awaited::Reclaim { .. }));
        assert!(start.elapsed() < Duration::from_secs(1));
    }

    /// While a receive is genuinely in flight, the wait must not expire out from under
    /// it -- proven with a short configured wait that a slower (but still eventually
    /// successful) receive still lands as `Arrived`.
    ///
    /// The waiter reports every pass of its wait loop through `cancelled()`, so the slot
    /// is only completed after the waiter has provably looped well past its 60 ms wait
    /// (one pass per 50 ms slice; without the extension it returns after about three),
    /// however late the thread was scheduled.
    #[test]
    fn a_receiving_upload_extends_the_deadline() {
        /// Wait passes that must be seen: at 50 ms per pass, far beyond the 60 ms wait.
        const PASSES: usize = 6;
        let range = SampleRange::new(0, 2);
        let slot = Arc::new(ContributionSlot::new(range, 1, 1));
        // Mark `Receiving` up front, exactly like `receive` does before it blocks on
        // I/O, and hold it there past the short wait below.
        slot.set_state(SlotState::Receiving);

        let (pass_tx, pass_rx) = mpsc::channel();
        let waiter = {
            let slot = Arc::clone(&slot);
            thread::spawn(move || {
                slot.await_until(Duration::from_millis(60), || {
                    let _ = pass_tx.send(());
                    false
                })
            })
        };
        for pass in 1..=PASSES {
            pass_rx
                .recv_timeout(Duration::from_secs(5))
                .unwrap_or_else(|_| {
                    panic!(
                        "the waiter stopped after {} of {PASSES} wait passes: its 60 ms wait \
                         expired while an upload was still being received",
                        pass - 1
                    )
                });
        }
        slot.set_state(SlotState::Arrived(vec![Vec3::ONE]));
        let outcome = waiter.join().unwrap();
        assert!(matches!(outcome, Awaited::Arrived(_)));
    }

    /// `cancelled()` ends the wait right away, even with time left.
    #[test]
    fn cancel_ends_the_wait() {
        let range = SampleRange::new(0, 2);
        let slot = ContributionSlot::new(range, 1, 1);
        let start = Instant::now();
        let outcome = slot.await_until(Duration::from_secs(30), || true);
        assert!(matches!(outcome, Awaited::Cancelled));
        assert!(start.elapsed() < Duration::from_secs(1));
    }
}