ic_testkit/pic/
transport.rs

1use std::{
2    any::Any,
3    panic::{AssertUnwindSafe, catch_unwind},
4};
5
6/// Captured PocketIC operation failure, retaining transport classification.
7#[derive(Clone, Debug, Eq, PartialEq)]
8pub struct PocketIcOperationError {
9    message: String,
10    transport: bool,
11}
12
13impl PocketIcOperationError {
14    /// Capture an upstream PocketIC panic message and classify its transport.
15    #[must_use]
16    pub fn new(message: impl Into<String>) -> Self {
17        let message = message.into();
18        let transport = is_dead_instance_transport_error(&message);
19        Self { message, transport }
20    }
21
22    /// Read the original upstream message without contextual wrappers.
23    #[must_use]
24    pub fn message(&self) -> &str {
25        &self.message
26    }
27
28    /// Whether the message matches a recognized PocketIC transport failure.
29    #[must_use]
30    pub const fn is_transport(&self) -> bool {
31        self.transport
32    }
33}
34
35// Only callers that intentionally capture every unwind use this boundary.
36// Candid calls retain their separate policy of resuming unrelated panics.
37pub(super) fn capture_operation<T>(
38    operation: impl FnOnce() -> T,
39) -> Result<T, PocketIcOperationError> {
40    catch_unwind(AssertUnwindSafe(operation))
41        .map_err(|payload| PocketIcOperationError::new(panic_payload_to_string(payload.as_ref())))
42}
43
44impl std::fmt::Display for PocketIcOperationError {
45    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
46        formatter.write_str(&self.message)
47    }
48}
49
50impl std::error::Error for PocketIcOperationError {}
51
52// Extract a stable string message from one panic payload.
53pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
54    if let Some(message) = payload.downcast_ref::<String>() {
55        return message.clone();
56    }
57    if let Some(message) = payload.downcast_ref::<&'static str>() {
58        return (*message).to_string();
59    }
60
61    "non-string panic payload".to_string()
62}
63
64// Check whether one panic payload belongs to the dead-instance transport class
65// without consuming it, so callers can still resume the original panic.
66pub(super) fn panic_is_dead_instance_transport(payload: &(dyn Any + Send)) -> bool {
67    payload
68        .downcast_ref::<String>()
69        .map(String::as_str)
70        .or_else(|| payload.downcast_ref::<&'static str>().copied())
71        .is_some_and(is_dead_instance_transport_error)
72}
73
74/// Recognize PocketIC transport failures in typed causes and request-error text.
75///
76/// The complete error source chain is inspected so a recipe can classify its
77/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
78/// preserves the classification already captured by testkit's structured errors.
79/// Testkit [`super::CandidCallError`], [`super::CanisterDiagnosticFailure`], and
80/// [`PocketIcOperationError`] causes retain their transport classification through
81/// contextual display wrappers, including nested [`std::io::Error`] values.
82/// Snapshot and installation failures expose their operation cause through
83/// the error source chain.
84///
85/// For other errors, message recognition requires a reqwest debug error with a
86/// PocketIC instance URL and a recognized transport source, optionally prefixed
87/// by PocketIC's HTTP panic context. Generic application messages and bare I/O
88/// error kinds do not qualify.
89/// Recognized sources include OS connection resets during an HTTP request.
90///
91/// Use this only for errors originating in a PocketIC operation. This is a
92/// message-based heuristic, not proof that the server or instance has died;
93/// another service or application can reproduce the same text. PocketIC does
94/// not yet expose a structured transport error.
95#[must_use]
96pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
97    let mut current = Some(error);
98    while let Some(candidate) = current {
99        if candidate
100            .downcast_ref::<PocketIcOperationError>()
101            .is_some_and(PocketIcOperationError::is_transport)
102        {
103            return true;
104        }
105        if candidate
106            .downcast_ref::<super::CandidCallError>()
107            .is_some_and(|error| matches!(error, super::CandidCallError::Transport { .. }))
108        {
109            return true;
110        }
111        if is_dead_instance_transport_error(&candidate.to_string()) {
112            return true;
113        }
114        // io::Error::source delegates to the contained error's source, skipping
115        // the contained error itself. Inspect it before following its causes so
116        // contextual errors keep their own structured classification.
117        current = if let Some(inner) = candidate
118            .downcast_ref::<std::io::Error>()
119            .and_then(std::io::Error::get_ref)
120        {
121            Some(inner)
122        } else {
123            candidate.source()
124        };
125    }
126    false
127}
128
129// Recognize maintained PocketIC request-error shapes for restore recovery.
130pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
131    let message = message
132        .strip_prefix("HTTP failure: ")
133        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
134        .unwrap_or(message);
135    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
136        return false;
137    };
138    let Some((kind, request)) = error.split_once(", url: \"") else {
139        return false;
140    };
141    if !matches!(kind, "Request" | "Body" | "Decode") {
142        return false;
143    }
144    let Some((url, source)) = request.split_once("\", source: ") else {
145        return false;
146    };
147    let Some(source) = source.strip_suffix(" }") else {
148        return false;
149    };
150    let Some(authority_and_path) = url
151        .strip_prefix("http://")
152        .or_else(|| url.strip_prefix("https://"))
153    else {
154        return false;
155    };
156    let Some((authority, path)) = authority_and_path.split_once('/') else {
157        return false;
158    };
159    let Some(instance_path) = path.strip_prefix("instances/") else {
160        return false;
161    };
162    let instance_id = instance_path.split('/').next().unwrap_or_default();
163    if authority.is_empty()
164        || instance_id.is_empty()
165        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
166    {
167        return false;
168    }
169    let source = source
170        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
171        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
172        .and_then(|source| source.strip_suffix(')'))
173        .unwrap_or(source);
174    let reset = source
175        .strip_prefix("hyper::Error(Io, Os { code: ")
176        .and_then(|source| source.split_once(", kind: ConnectionReset, message: \""))
177        .is_some_and(|(code, message)| code.parse::<i32>().is_ok() && message.ends_with("\" })"));
178    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
179        || reset
180        || matches!(
181            source,
182            "hyper::Error(IncompleteMessage)"
183                | "hyper::Error(ChannelClosed)"
184                | "hyper::Error(Canceled, \"connection closed before message completed\")"
185        )
186}
187
188#[cfg(test)]
189mod tests {
190    use super::super::{CanisterInstallError, ControllerSnapshotError};
191    use super::{
192        PocketIcOperationError, capture_operation, is_dead_instance_transport_error,
193        is_dead_pocket_ic_transport_error, panic_is_dead_instance_transport,
194    };
195
196    #[derive(Debug)]
197    struct WrapperError(std::io::Error);
198
199    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
200    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
201    const RESET: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:49238/instances/0/update/submit_ingress_message\", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(Io, Os { code: 54, kind: ConnectionReset, message: \"Connection reset by peer\" })) }";
202
203    impl std::fmt::Display for WrapperError {
204        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
205            formatter.write_str("wrapped PocketIC request failed")
206        }
207    }
208
209    impl std::error::Error for WrapperError {
210        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
211            Some(&self.0)
212        }
213    }
214
215    #[test]
216    fn dead_instance_transport_error_detects_connection_refused() {
217        assert!(is_dead_instance_transport_error(REFUSED));
218        assert!(is_dead_instance_transport_error(&format!(
219            "HTTP failure: {REFUSED}"
220        )));
221    }
222
223    #[test]
224    fn dead_instance_transport_error_detects_incomplete_message() {
225        assert!(is_dead_instance_transport_error(INCOMPLETE));
226        assert!(is_dead_instance_transport_error(&format!(
227            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
228        )));
229    }
230
231    #[test]
232    fn dead_instance_transport_error_detects_native_connection_reset() {
233        for message in [RESET.to_owned(), RESET.replace("code: 54", "code: 104")] {
234            assert!(is_dead_instance_transport_error(&message));
235            assert!(panic_is_dead_instance_transport(&format!(
236                "HTTP failure: {message}"
237            )));
238            assert!(is_dead_instance_transport_error(&format!(
239                "called `Result::unwrap()` on an `Err` value: {message}"
240            )));
241        }
242    }
243
244    #[test]
245    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
246        let classified = capture_operation::<()>(|| panic!("{INCOMPLETE}"))
247            .expect_err("operation panic must be captured");
248
249        assert_eq!(classified.message(), INCOMPLETE);
250        assert!(classified.is_transport());
251        assert!(panic_is_dead_instance_transport(&INCOMPLETE));
252        let non_string = capture_operation::<()>(|| std::panic::panic_any(7_u8))
253            .expect_err("non-string panic must be captured");
254        assert_eq!(non_string.message(), "non-string panic payload");
255        assert!(!non_string.is_transport());
256    }
257
258    #[test]
259    fn public_classifier_inspects_the_error_source_chain() {
260        let dead = WrapperError(std::io::Error::other(REFUSED));
261        assert!(is_dead_pocket_ic_transport_error(&dead));
262
263        let unrelated = WrapperError(std::io::Error::other("request rejected"));
264        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
265    }
266
267    #[test]
268    fn snapshot_and_install_wrappers_preserve_transport_causes() {
269        let canister_id = candid::Principal::anonymous();
270        for message in [REFUSED, RESET, "unrelated application panic"] {
271            let expected = message != "unrelated application panic";
272            let capture = ControllerSnapshotError::CapturePanicked {
273                canister_id,
274                source: PocketIcOperationError::new(message),
275                cleanup_failures: vec![],
276            };
277            let restore = ControllerSnapshotError::RestorePanicked {
278                canister_id,
279                source: PocketIcOperationError::new(message),
280            };
281            for snapshot in [capture, restore] {
282                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
283                let wrapper = WrapperError(std::io::Error::other(snapshot));
284                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
285            }
286            for install in [
287                CanisterInstallError::CreateCanister {
288                    label: Some("fixture".into()),
289                    source: PocketIcOperationError::new(message),
290                },
291                CanisterInstallError::AddCycles {
292                    canister_id,
293                    label: Some("fixture".into()),
294                    source: PocketIcOperationError::new(message),
295                },
296                CanisterInstallError::InstallCode {
297                    canister_id,
298                    label: Some("fixture".into()),
299                    source: PocketIcOperationError::new(message),
300                },
301            ] {
302                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
303                let wrapper = WrapperError(std::io::Error::other(install));
304                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
305            }
306        }
307    }
308
309    #[test]
310    fn unrelated_and_quoted_transport_text_does_not_qualify() {
311        for message in [
312            "application worker channel closed",
313            "fixture expected ConnectionRefused but observed another result",
314            "tcp connect error",
315            "connection closed before message completed",
316            "IncompleteMessage",
317            &format!("fixture quoted {REFUSED}"),
318            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
319            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
320            &INCOMPLETE.replace(
321                "hyper::Error(IncompleteMessage)",
322                "Custom(\"channel closed\")",
323            ),
324            &INCOMPLETE.replace(
325                "hyper::Error(IncompleteMessage)",
326                "Custom(\"hyper::Error(IncompleteMessage)\")",
327            ),
328            &format!("fixture quoted {RESET}"),
329            &RESET.replace("/instances/0/", "/application/worker/"),
330            &RESET.replace("/instances/0/", "/instances/quoted/"),
331            &RESET.replace("kind: ConnectionReset,", "kind: ConnectionAborted,"),
332            &RESET.replace("code: 54", "code: quoted"),
333            &RESET.replace("hyper::Error(Io, Os {", "Custom(Os {"),
334        ] {
335            let error = WrapperError(std::io::Error::other(message.to_owned()));
336            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
337            assert!(!PocketIcOperationError::new(message).is_transport());
338            assert!(!panic_is_dead_instance_transport(&message.to_owned()));
339        }
340        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
341        assert!(!is_dead_pocket_ic_transport_error(&refused));
342        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
343        let reset = std::io::Error::from(std::io::ErrorKind::ConnectionReset);
344        assert!(!is_dead_pocket_ic_transport_error(&reset));
345        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(reset)));
346    }
347
348    #[test]
349    fn structured_channel_closure_on_an_instance_request_qualifies() {
350        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
351        assert!(is_dead_instance_transport_error(&message));
352    }
353
354    #[test]
355    fn contextual_call_errors_use_their_structured_kind() {
356        let context = super::super::CandidCallContext::new(
357            "query_call",
358            candid::Principal::anonymous(),
359            candid::Principal::anonymous(),
360            "get",
361        );
362        let error = super::super::CandidCallError::Transport {
363            context: Box::new(context.clone()),
364            message: REFUSED.into(),
365        };
366        assert!(is_dead_pocket_ic_transport_error(&error));
367        let error = super::super::CandidCallError::Decode {
368            context: Box::new(context),
369            bytes: 0,
370            message: "channel closed".into(),
371        };
372        assert!(!is_dead_pocket_ic_transport_error(&error));
373    }
374
375    #[test]
376    fn io_wrappers_preserve_contextual_call_error_kinds() {
377        let context = super::super::CandidCallContext::new(
378            "query_call",
379            candid::Principal::anonymous(),
380            candid::Principal::anonymous(),
381            "get",
382        );
383        for (error, expected) in [
384            (
385                super::super::CandidCallError::Transport {
386                    context: Box::new(context.clone()),
387                    message: REFUSED.into(),
388                },
389                true,
390            ),
391            (
392                super::super::CandidCallError::Decode {
393                    context: Box::new(context),
394                    bytes: 0,
395                    message: REFUSED.into(),
396                },
397                false,
398            ),
399        ] {
400            let inner = std::io::Error::other(error);
401            assert_eq!(is_dead_pocket_ic_transport_error(&inner), expected);
402            let wrapped = WrapperError(std::io::Error::other(inner));
403            assert_eq!(is_dead_pocket_ic_transport_error(&wrapped), expected);
404        }
405    }
406}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
