ice-rpc 0.3.6

RPC over iceoryx2 shared memory: zero-copy header, rkyv payload, reconnection, discovery and more
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
//! Opening an iceoryx2 service, and what its failure means.
//!
//! iceoryx2 records the **static configuration** of a service when it is created
//! — wire format, payload alignment, buffer sizes, port limits — and refuses to
//! open it later if a process asks for a different one. A process killed while it
//! held a service leaves the same kind of state behind. Both cases look like a
//! transport failure from the call site, and neither is one: retrying changes
//! nothing, and the raw `Debug` of the iceoryx2 error tells the reader nothing
//! about what to do.
//!
//! The two cases have **opposite** remedies, so this module tells them apart: a
//! configuration mismatch is fixed by aligning the participants'
//! `iceoryx2.toml`, leftover state by removing the iceoryx2 root path. Confusing
//! the two sends the reader to a purge that cannot help — the same divergent
//! configuration comes back on the next start.

use iceoryx2::prelude::*;
use iceoryx2::service::builder::event::{EventCreateError, EventOpenError, EventOpenOrCreateError};
use iceoryx2::service::builder::publish_subscribe::{
    PublishSubscribeCreateError, PublishSubscribeOpenError, PublishSubscribeOpenOrCreateError,
};

use super::{IoxEvent, IoxNode, IoxPubSub, OPEN_RETRY_SLEEP, PAYLOAD_ALIGNMENT, transport_error};
use crate::CancellationToken;
use crate::types::{RpcError, RpcHeader};

/// Whether opening a service may create it.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum OpenMode {
    /// Transport side: create the service when no peer owns it yet.
    CreateOrOpen,
    /// Observer side: attach to an existing service, never create one.
    ReadOnly,
}

/// What to do when the peers disagree on a service's configuration.
///
/// The service limits are no longer pinned by the library: the deployment owns
/// them through the iceoryx2 configuration, so this failure is a deployment
/// concern, not leftover state — removing the iceoryx2 root path would not help,
/// because the same divergent configuration comes back on the next start.
const REMEDY_CONFIG: &str = "the iceoryx2 configuration resolved by this process differs from the \
                             one recorded when the service was created: payload alignment, \
                             overflow behavior, buffer sizes or port limits. Every participant of \
                             this channel must resolve the same iceoryx2 configuration (notably \
                             its `[defaults.publish-subscribe]`) and speak the same protocol \
                             version";

/// What to do when the bus state itself is unusable.
///
/// A process killed while it held the service leaves an entry iceoryx2 can no
/// longer reconcile. The configuration may be right, so restarting will not help
/// until the leftover state is removed.
const REMEDY_STALE: &str = "the iceoryx2 state on this machine was left behind by a process that \
                            was killed while it held this service. Once no process still runs, \
                            remove the iceoryx2 root path (the `root-path` of the effective \
                            `iceoryx2.toml`, or iceoryx2's default: C:\\Temp\\iceoryx2 on Windows, \
                            /tmp/iceoryx2 elsewhere)";

/// Opens the pub/sub service of one direction of `channel`.
///
/// The definition is shared by the transport and the observer, so an
/// out-of-band observer is guaranteed to attach to the very service the
/// transport created.
pub(super) fn open_service(
    node: &IoxNode,
    channel: &str,
    suffix: &str,
    mode: OpenMode,
) -> Result<IoxPubSub, RpcError> {
    let topic = format!("{channel}{suffix}");
    let name = ServiceName::new(&topic).map_err(|e| transport_error("service name", e))?;
    let alignment = Alignment::new(PAYLOAD_ALIGNMENT)
        .ok_or_else(|| RpcError::Internal("invalid payload alignment".to_string()))?;
    let builder = node
        .service_builder(&name)
        .publish_subscribe::<[u8]>()
        .user_header::<RpcHeader>()
        .payload_alignment(alignment)
        // Must stay false: enabled, the receiver overwrites its oldest sample.
        .enable_safe_overflow(false);

    match mode {
        OpenMode::CreateOrOpen => builder.open_or_create().map_err(|e| match e {
            PublishSubscribeOpenOrCreateError::PublishSubscribeOpenError(inner) => {
                pub_sub_open_error("open service", inner)
            }
            PublishSubscribeOpenOrCreateError::PublishSubscribeCreateError(inner) => {
                pub_sub_create_error("create service", inner)
            }
            PublishSubscribeOpenOrCreateError::SystemInFlux => system_in_flux("open service"),
        }),
        OpenMode::ReadOnly => builder
            .open()
            .map_err(|e| pub_sub_open_error("open service (read-only)", e)),
    }
}

/// Opens the event service used as a wake-up signal for `channel`.
pub(super) fn open_event_service(
    node: &IoxNode,
    channel: &str,
    suffix: &str,
    mode: OpenMode,
) -> Result<IoxEvent, RpcError> {
    let topic = format!("{channel}{suffix}");
    let name = ServiceName::new(&topic).map_err(|e| transport_error("service name", e))?;
    let builder = node.service_builder(&name).event();

    match mode {
        OpenMode::CreateOrOpen => builder.open_or_create().map_err(event_error),
        OpenMode::ReadOnly => builder
            .open()
            .map_err(|e| event_open_error("open event service (read-only)", e)),
    }
}

/// Retries `attempt_open` while it fails with a retryable error, until it
/// succeeds, the budget runs out, or `stop` is cancelled.
///
/// iceoryx2 classifies the transient races — `SystemInFlux`, and the
/// `InternalFailure` a concurrent create yields on Windows — as retryable; this
/// loop is what makes that true, for the provider opening on its own thread and
/// the consumer opening on its first call. A non-retryable failure returns at
/// once, remedy included, and a cancelled `stop` returns after the sleep in
/// progress.
pub(super) fn retry_open<T>(
    channel: &str,
    attempts: u32,
    stop: &CancellationToken,
    mut attempt_open: impl FnMut() -> Result<T, RpcError>,
) -> Result<T, RpcError> {
    let mut attempt = 0;
    loop {
        match attempt_open() {
            Ok(opened) => return Ok(opened),
            Err(e) if !e.is_retryable() || attempt >= attempts || stop.is_cancelled() => {
                return Err(e);
            }
            Err(e) => {
                attempt += 1;
                log::warn!(
                    "[transport] '{channel}': {e} — attempt {attempt}/{attempts}, retrying in {:?}",
                    OPEN_RETRY_SLEEP
                );
                std::thread::sleep(OPEN_RETRY_SLEEP);
            }
        }
    }
}

/// Whether the failure means "the peers disagree on this service's configuration".
///
/// The `Incompatible*` variants are the identity properties — wire types,
/// messaging pattern, overflow behavior. They are pinned protocol invariants, so
/// they can only differ between two builds that disagree on the protocol. The
/// `DoesNotSupportRequested*` variants are the capacity properties, which the
/// deployment now owns through the iceoryx2 configuration: they mean a peer
/// asked for more than the creator of the service had reserved.
fn is_config_mismatch(error: &PublishSubscribeOpenError) -> bool {
    use PublishSubscribeOpenError as E;
    matches!(
        error,
        E::IncompatibleTypes
            | E::IncompatibleMessagingPattern
            | E::IncompatibleAttributes
            | E::IncompatibleOverflowBehavior
            | E::DoesNotSupportRequestedMinBufferSize
            | E::DoesNotSupportRequestedMinHistorySize
            | E::DoesNotSupportRequestedMinSubscriberBorrowedSamples
            | E::DoesNotSupportRequestedAmountOfPublishers
            | E::DoesNotSupportRequestedAmountOfSubscribers
            | E::DoesNotSupportRequestedAmountOfNodes
    )
}

/// Whether the failure means "this machine holds state a killed process left".
///
/// The two variants are the ones describing state iceoryx2 can no longer
/// reconcile: resources missing or corrupted, and a creation that never finished
/// because its process died.
fn is_stale_state(error: &PublishSubscribeOpenError) -> bool {
    matches!(
        error,
        PublishSubscribeOpenError::ServiceInCorruptedState
            | PublishSubscribeOpenError::HangsInCreation
    )
}

/// Same question as [`is_stale_state`], for the creation side.
///
/// Creating a service never reports a configuration mismatch — the creator
/// defines the configuration — so only leftover state can fail it.
fn is_stale_creation(error: &PublishSubscribeCreateError) -> bool {
    matches!(
        error,
        PublishSubscribeCreateError::ServiceInCorruptedState
            | PublishSubscribeCreateError::HangsInCreation
    )
}

/// Reports a pub/sub open failure: a configuration mismatch or leftover state is
/// a protocol mismatch, the rest is a transport failure.
fn pub_sub_open_error(context: &str, error: PublishSubscribeOpenError) -> RpcError {
    if is_config_mismatch(&error) {
        RpcError::ProtocolMismatch(format!("{context}: {error:?}. {REMEDY_CONFIG}"))
    } else if is_stale_state(&error) {
        RpcError::ProtocolMismatch(format!("{context}: {error:?}. {REMEDY_STALE}"))
    } else {
        transport_error(context, error)
    }
}

/// Reports a pub/sub creation failure, with the same split.
fn pub_sub_create_error(context: &str, error: PublishSubscribeCreateError) -> RpcError {
    if is_stale_creation(&error) {
        RpcError::ProtocolMismatch(format!("{context}: {error:?}. {REMEDY_STALE}"))
    } else {
        transport_error(context, error)
    }
}

/// Reports an event-service failure, with the same split.
fn event_error(error: EventOpenOrCreateError) -> RpcError {
    match error {
        EventOpenOrCreateError::EventOpenError(inner) => event_open_error("open event", inner),
        EventOpenOrCreateError::EventCreateError(inner) => {
            event_create_error("create event", inner)
        }
        EventOpenOrCreateError::SystemInFlux => system_in_flux("open event service"),
    }
}

/// Message for iceoryx2's `SystemInFlux`, which has two very different causes.
///
/// It stays a [`RpcError::TransportError`], hence retryable, because one of the
/// causes is genuinely transient: another process is creating or removing that
/// exact service at this instant, and the next attempt succeeds.
///
/// The other cause is not: a process that died while it held the service leaves
/// an entry iceoryx2 can no longer reconcile — the same shape as
/// `ServiceInCorruptedState` and `HangsInCreation`, which are already reported as
/// a [`RpcError::ProtocolMismatch`] with the remedy. A bare `SystemInFlux` in a
/// log leaves the reader to guess which of the two they have, and the case it
/// hides is the one that never resolves itself, so the message names both and
/// carries the remedy.
fn system_in_flux(context: &str) -> RpcError {
    RpcError::TransportError(format!(
        "{context}: SystemInFlux. Either another process is creating or removing this service \
         right now, in which case retrying works, or the state is left over from a process that \
         died while it held it, in which case retrying cannot help and this persists: {REMEDY_STALE}"
    ))
}

/// Reports an event-service creation failure, with the same split.
fn event_create_error(context: &str, error: EventCreateError) -> RpcError {
    if matches!(error, EventCreateError::ServiceInCorruptedState) {
        RpcError::ProtocolMismatch(format!("{context}: {error:?}. {REMEDY_STALE}"))
    } else {
        transport_error(context, error)
    }
}

/// Reports an event-service open failure, with the same split.
fn event_open_error(context: &str, error: EventOpenError) -> RpcError {
    use EventOpenError as E;
    let config_mismatch = matches!(
        error,
        E::IncompatibleMessagingPattern
            | E::IncompatibleAttributes
            | E::IncompatibleDeadline
            | E::IncompatibleNotifierCreatedEvent
            | E::IncompatibleNotifierDroppedEvent
            | E::IncompatibleNotifierDeadEvent
            | E::DoesNotSupportRequestedAmountOfNotifiers
            | E::DoesNotSupportRequestedAmountOfListeners
            | E::DoesNotSupportRequestedMaxEventId
            | E::DoesNotSupportRequestedAmountOfNodes
    );

    if config_mismatch {
        RpcError::ProtocolMismatch(format!("{context}: {error:?}. {REMEDY_CONFIG}"))
    } else if matches!(error, E::ServiceInCorruptedState | E::HangsInCreation) {
        RpcError::ProtocolMismatch(format!("{context}: {error:?}. {REMEDY_STALE}"))
    } else {
        transport_error(context, error)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use iceoryx2::service::builder::event::EventOpenError;
    use iceoryx2::service::builder::publish_subscribe::PublishSubscribeOpenError;

    /// A service whose recorded configuration is not the requested one is a
    /// protocol mismatch whose remedy is a shared configuration — not a purge.
    #[test]
    fn a_configuration_mismatch_is_reported_as_a_protocol_mismatch() {
        for error in [
            PublishSubscribeOpenError::IncompatibleTypes,
            PublishSubscribeOpenError::IncompatibleOverflowBehavior,
            PublishSubscribeOpenError::DoesNotSupportRequestedMinBufferSize,
            PublishSubscribeOpenError::DoesNotSupportRequestedAmountOfPublishers,
        ] {
            let mapped = pub_sub_open_error("open service", error);
            assert!(
                matches!(&mapped, RpcError::ProtocolMismatch(_)),
                "{mapped:?}"
            );
            assert!(
                !mapped.is_retryable(),
                "no retry fixes a configuration mismatch"
            );
            let text = mapped.to_string();
            assert!(text.contains("configuration"), "{text}");
            assert!(
                !text.contains("root path"),
                "a config mismatch is not fixed by a purge: {text}"
            );
        }
    }

    /// State left behind by a killed process is also a protocol mismatch, but its
    /// remedy is the purge of the iceoryx2 root path.
    #[test]
    fn stale_state_is_reported_with_the_purge_remedy() {
        for error in [
            PublishSubscribeOpenError::ServiceInCorruptedState,
            PublishSubscribeOpenError::HangsInCreation,
        ] {
            let mapped = pub_sub_open_error("open service", error);
            assert!(
                matches!(&mapped, RpcError::ProtocolMismatch(_)),
                "{mapped:?}"
            );
            assert!(!mapped.is_retryable(), "no retry fixes stale state");
            assert!(mapped.to_string().contains("root path"), "{mapped}");
        }
    }

    /// Everything else stays a transport failure, so the retry policy of the
    /// caller does not change for a genuinely transient error.
    #[test]
    fn another_open_failure_stays_a_transport_error() {
        let mapped = pub_sub_open_error("open service", PublishSubscribeOpenError::DoesNotExist);
        assert!(matches!(&mapped, RpcError::TransportError(_)), "{mapped:?}");
        assert!(mapped.is_retryable());
    }

    /// `SystemInFlux` covers both a concurrent create and left-over state: it
    /// must stay retryable for the first, and say what to do about the second.
    #[test]
    fn system_in_flux_names_both_of_its_causes() {
        let mapped = system_in_flux("open service");

        assert!(matches!(&mapped, RpcError::TransportError(_)), "{mapped:?}");
        assert!(
            mapped.is_retryable(),
            "a concurrent create is transient, so the retry policy must not change"
        );

        let text = mapped.to_string();
        assert!(text.contains("retrying works"), "{text}");
        assert!(
            text.contains("root path"),
            "the persistent case must carry the remedy: {text}"
        );
    }

    /// The event services (the wake-up channels) are classified the same way.
    #[test]
    fn the_event_side_is_classified_too() {
        let stale = event_open_error(
            "open event",
            EventOpenError::IncompatibleNotifierCreatedEvent,
        );
        assert!(matches!(&stale, RpcError::ProtocolMismatch(_)), "{stale:?}");

        let transient = event_open_error("open event", EventOpenError::DoesNotExist);
        assert!(
            matches!(&transient, RpcError::TransportError(_)),
            "{transient:?}"
        );
    }

    /// The open-or-create dispatch must route each cause to its own classifier,
    /// whichever half of the pair iceoryx2 reported.
    #[test]
    fn the_open_or_create_event_error_is_dispatched_by_cause() {
        let opened = event_error(EventOpenOrCreateError::EventOpenError(
            EventOpenError::DoesNotExist,
        ));
        assert!(matches!(&opened, RpcError::TransportError(_)), "{opened:?}");

        let created = event_error(EventOpenOrCreateError::EventCreateError(
            EventCreateError::ServiceInCorruptedState,
        ));
        assert!(
            matches!(&created, RpcError::ProtocolMismatch(_)),
            "{created:?}"
        );
        assert!(created.to_string().contains("root path"), "{created}");

        let flux = event_error(EventOpenOrCreateError::SystemInFlux);
        assert!(matches!(&flux, RpcError::TransportError(_)), "{flux:?}");
        assert!(flux.is_retryable(), "a concurrent create is transient");
    }

    /// Creating a service never reports a configuration mismatch — the creator
    /// defines the configuration — so only leftover state is a protocol mismatch
    /// and everything else stays a transport failure.
    #[test]
    fn a_creation_failure_only_blames_the_peers_for_stale_state() {
        let stale = pub_sub_create_error(
            "create service",
            PublishSubscribeCreateError::ServiceInCorruptedState,
        );
        assert!(matches!(&stale, RpcError::ProtocolMismatch(_)), "{stale:?}");
        assert!(stale.to_string().contains("root path"), "{stale}");

        let transient = pub_sub_create_error(
            "create service",
            PublishSubscribeCreateError::InsufficientPermissions,
        );
        assert!(
            matches!(&transient, RpcError::TransportError(_)),
            "{transient:?}"
        );

        // The event side has the same split.
        let event_stale =
            event_create_error("create event", EventCreateError::ServiceInCorruptedState);
        assert!(
            matches!(&event_stale, RpcError::ProtocolMismatch(_)),
            "{event_stale:?}"
        );
        assert!(
            event_stale.to_string().contains("root path"),
            "{event_stale}"
        );

        let event_transient =
            event_create_error("create event", EventCreateError::InsufficientPermissions);
        assert!(
            matches!(&event_transient, RpcError::TransportError(_)),
            "{event_transient:?}"
        );
    }

    /// A token nobody cancelled: the retries run their course.
    fn running() -> CancellationToken {
        CancellationToken::new()
    }

    /// The race `SystemInFlux` describes is transient: it must be retried, and
    /// the loop must return what the successful attempt produced.
    #[test]
    fn a_retryable_open_failure_is_retried_until_it_succeeds() {
        let mut attempts = 0;
        let opened = retry_open("chan", 5, &running(), || {
            attempts += 1;
            if attempts < 3 {
                Err(RpcError::TransportError("SystemInFlux".to_string()))
            } else {
                Ok(attempts)
            }
        });

        assert_eq!(opened.unwrap(), 3);
        assert_eq!(attempts, 3, "two failures then one success");
    }

    /// A service left behind by another build cannot be fixed by retrying: the
    /// caller must get the error — and the remedy it carries — immediately.
    #[test]
    fn a_stale_open_failure_is_not_retried() {
        let mut attempts = 0;
        let opened = retry_open::<()>("chan", 5, &running(), || {
            attempts += 1;
            Err(RpcError::ProtocolMismatch("stale service".to_string()))
        });

        assert!(opened.is_err());
        assert_eq!(attempts, 1, "not one wasted attempt on a stale service");
    }

    /// The budget is a budget: it is spent, then the error comes out.
    #[test]
    fn the_retry_budget_is_finite() {
        let mut attempts = 0;
        let opened = retry_open::<()>("chan", 2, &running(), || {
            attempts += 1;
            Err(RpcError::TransportError("SystemInFlux".to_string()))
        });

        assert!(matches!(opened, Err(RpcError::TransportError(_))));
        assert_eq!(attempts, 3, "the first attempt plus the two retries");
    }

    /// A shutdown request ends the wait at once: the worst case of a Ctrl+C is
    /// the sleep in progress, not the whole budget.
    #[test]
    fn a_cancelled_shutdown_stops_the_retries() {
        let stop = CancellationToken::new();
        stop.cancel();

        let mut attempts = 0;
        let opened = retry_open::<()>("chan", 5, &stop, || {
            attempts += 1;
            Err(RpcError::TransportError("SystemInFlux".to_string()))
        });

        assert!(opened.is_err());
        assert_eq!(attempts, 1, "a cancelled token stops before any retry");
    }
}