ic_testkit/pic/
transport.rs

1use std::{
2    any::Any,
3    panic::{AssertUnwindSafe, catch_unwind},
4};
5
6/// Captured PocketIC operation failure, retaining transport classification.
7#[derive(Clone, Debug, Eq, PartialEq)]
8pub struct PocketIcOperationError {
9    message: String,
10    transport: bool,
11}
12
13impl PocketIcOperationError {
14    /// Capture an upstream PocketIC panic message and classify its transport.
15    #[must_use]
16    pub fn new(message: impl Into<String>) -> Self {
17        let message = message.into();
18        let transport = is_dead_instance_transport_error(&message);
19        Self { message, transport }
20    }
21
22    /// Read the original upstream message without contextual wrappers.
23    #[must_use]
24    pub fn message(&self) -> &str {
25        &self.message
26    }
27
28    /// Whether the message matches a recognized PocketIC transport failure.
29    #[must_use]
30    pub const fn is_transport(&self) -> bool {
31        self.transport
32    }
33}
34
35// Only callers that intentionally capture every unwind use this boundary.
36// Candid calls retain their separate policy of resuming unrelated panics.
37pub(super) fn capture_operation<T>(
38    operation: impl FnOnce() -> T,
39) -> Result<T, PocketIcOperationError> {
40    catch_unwind(AssertUnwindSafe(operation))
41        .map_err(|payload| PocketIcOperationError::new(panic_payload_to_string(payload.as_ref())))
42}
43
44impl std::fmt::Display for PocketIcOperationError {
45    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
46        formatter.write_str(&self.message)
47    }
48}
49
50impl std::error::Error for PocketIcOperationError {}
51
52// Extract a stable string message from one panic payload.
53pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
54    if let Some(message) = payload.downcast_ref::<String>() {
55        return message.clone();
56    }
57    if let Some(message) = payload.downcast_ref::<&'static str>() {
58        return (*message).to_string();
59    }
60
61    "non-string panic payload".to_string()
62}
63
64// Check whether one panic payload belongs to the dead-instance transport class
65// without consuming it, so callers can still resume the original panic.
66pub(super) fn panic_is_dead_instance_transport(payload: &(dyn Any + Send)) -> bool {
67    payload
68        .downcast_ref::<String>()
69        .map(String::as_str)
70        .or_else(|| payload.downcast_ref::<&'static str>().copied())
71        .is_some_and(is_dead_instance_transport_error)
72}
73
74/// Recognize PocketIC transport failures in typed causes and request-error text.
75///
76/// The complete error source chain is inspected so a recipe can classify its
77/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
78/// preserves the classification already captured by testkit's structured errors.
79/// Testkit [`super::CandidCallError`], [`super::CanisterDiagnosticFailure`], and
80/// [`PocketIcOperationError`] causes retain their transport classification through
81/// contextual display wrappers, including nested [`std::io::Error`] values.
82/// Snapshot and installation failures expose their operation cause through
83/// the error source chain.
84///
85/// For other errors, message recognition requires a reqwest debug error with a
86/// PocketIC instance URL and a recognized transport source, optionally prefixed
87/// by PocketIC's HTTP panic context. Generic application messages and bare I/O
88/// error kinds do not qualify.
89/// Recognized sources include OS connection resets during an HTTP request.
90///
91/// Use this only for errors originating in a PocketIC operation. This is a
92/// message-based heuristic, not proof that the server or instance has died;
93/// another service or application can reproduce the same text. PocketIC does
94/// not yet expose a structured transport error.
95#[must_use]
96pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
97    let mut current = Some(error);
98    while let Some(candidate) = current {
99        if candidate
100            .downcast_ref::<PocketIcOperationError>()
101            .is_some_and(PocketIcOperationError::is_transport)
102        {
103            return true;
104        }
105        if candidate
106            .downcast_ref::<super::CandidCallError>()
107            .is_some_and(|error| error.kind == super::CandidCallErrorKind::Transport)
108        {
109            return true;
110        }
111        if matches!(
112            candidate.downcast_ref::<super::CanisterDiagnosticFailure>(),
113            Some(super::CanisterDiagnosticFailure::InstanceUnavailable { .. })
114        ) {
115            return true;
116        }
117        if is_dead_instance_transport_error(&candidate.to_string()) {
118            return true;
119        }
120        // io::Error::source delegates to the contained error's source, skipping
121        // the contained error itself. Inspect it before following its causes so
122        // contextual errors keep their own structured classification.
123        current = if let Some(inner) = candidate
124            .downcast_ref::<std::io::Error>()
125            .and_then(std::io::Error::get_ref)
126        {
127            Some(inner)
128        } else {
129            candidate.source()
130        };
131    }
132    false
133}
134
135// Recognize maintained PocketIC request-error shapes for restore recovery.
136pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
137    let message = message
138        .strip_prefix("HTTP failure: ")
139        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
140        .unwrap_or(message);
141    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
142        return false;
143    };
144    let Some((kind, request)) = error.split_once(", url: \"") else {
145        return false;
146    };
147    if !matches!(kind, "Request" | "Body" | "Decode") {
148        return false;
149    }
150    let Some((url, source)) = request.split_once("\", source: ") else {
151        return false;
152    };
153    let Some(source) = source.strip_suffix(" }") else {
154        return false;
155    };
156    let Some(authority_and_path) = url
157        .strip_prefix("http://")
158        .or_else(|| url.strip_prefix("https://"))
159    else {
160        return false;
161    };
162    let Some((authority, path)) = authority_and_path.split_once('/') else {
163        return false;
164    };
165    let Some(instance_path) = path.strip_prefix("instances/") else {
166        return false;
167    };
168    let instance_id = instance_path.split('/').next().unwrap_or_default();
169    if authority.is_empty()
170        || instance_id.is_empty()
171        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
172    {
173        return false;
174    }
175    let source = source
176        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
177        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
178        .and_then(|source| source.strip_suffix(')'))
179        .unwrap_or(source);
180    let reset = source
181        .strip_prefix("hyper::Error(Io, Os { code: ")
182        .and_then(|source| source.split_once(", kind: ConnectionReset, message: \""))
183        .is_some_and(|(code, message)| code.parse::<i32>().is_ok() && message.ends_with("\" })"));
184    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
185        || reset
186        || matches!(
187            source,
188            "hyper::Error(IncompleteMessage)"
189                | "hyper::Error(ChannelClosed)"
190                | "hyper::Error(Canceled, \"connection closed before message completed\")"
191        )
192}
193
194#[cfg(test)]
195mod tests {
196    use super::super::{CanisterInstallError, CanisterInstallPhase, ControllerSnapshotError};
197    use super::{
198        PocketIcOperationError, capture_operation, is_dead_instance_transport_error,
199        is_dead_pocket_ic_transport_error, panic_is_dead_instance_transport,
200    };
201
202    #[derive(Debug)]
203    struct WrapperError(std::io::Error);
204
205    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
206    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
207    const RESET: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:49238/instances/0/update/submit_ingress_message\", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(Io, Os { code: 54, kind: ConnectionReset, message: \"Connection reset by peer\" })) }";
208
209    impl std::fmt::Display for WrapperError {
210        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
211            formatter.write_str("wrapped PocketIC request failed")
212        }
213    }
214
215    impl std::error::Error for WrapperError {
216        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
217            Some(&self.0)
218        }
219    }
220
221    #[test]
222    fn dead_instance_transport_error_detects_connection_refused() {
223        assert!(is_dead_instance_transport_error(REFUSED));
224        assert!(is_dead_instance_transport_error(&format!(
225            "HTTP failure: {REFUSED}"
226        )));
227    }
228
229    #[test]
230    fn dead_instance_transport_error_detects_incomplete_message() {
231        assert!(is_dead_instance_transport_error(INCOMPLETE));
232        assert!(is_dead_instance_transport_error(&format!(
233            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
234        )));
235    }
236
237    #[test]
238    fn dead_instance_transport_error_detects_native_connection_reset() {
239        for message in [RESET.to_owned(), RESET.replace("code: 54", "code: 104")] {
240            assert!(is_dead_instance_transport_error(&message));
241            assert!(panic_is_dead_instance_transport(&format!(
242                "HTTP failure: {message}"
243            )));
244            assert!(is_dead_instance_transport_error(&format!(
245                "called `Result::unwrap()` on an `Err` value: {message}"
246            )));
247        }
248    }
249
250    #[test]
251    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
252        let classified = capture_operation::<()>(|| panic!("{INCOMPLETE}"))
253            .expect_err("operation panic must be captured");
254
255        assert_eq!(classified.message(), INCOMPLETE);
256        assert!(classified.is_transport());
257        assert!(panic_is_dead_instance_transport(&INCOMPLETE));
258        let non_string = capture_operation::<()>(|| std::panic::panic_any(7_u8))
259            .expect_err("non-string panic must be captured");
260        assert_eq!(non_string.message(), "non-string panic payload");
261        assert!(!non_string.is_transport());
262    }
263
264    #[test]
265    fn public_classifier_inspects_the_error_source_chain() {
266        let dead = WrapperError(std::io::Error::other(REFUSED));
267        assert!(is_dead_pocket_ic_transport_error(&dead));
268
269        let unrelated = WrapperError(std::io::Error::other("request rejected"));
270        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
271    }
272
273    #[test]
274    fn snapshot_and_install_wrappers_preserve_transport_causes() {
275        let canister_id = candid::Principal::anonymous();
276        for message in [REFUSED, RESET, "unrelated application panic"] {
277            let expected = message != "unrelated application panic";
278            let capture = ControllerSnapshotError::CapturePanicked {
279                canister_id,
280                source: PocketIcOperationError::new(message),
281                cleanup_failures: vec![],
282            };
283            let restore = ControllerSnapshotError::RestorePanicked {
284                canister_id,
285                source: PocketIcOperationError::new(message),
286            };
287            for snapshot in [capture, restore] {
288                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
289                let wrapper = WrapperError(std::io::Error::other(snapshot));
290                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
291            }
292            for phase in [
293                CanisterInstallPhase::CreateCanister,
294                CanisterInstallPhase::AddCycles,
295                CanisterInstallPhase::InstallCode,
296            ] {
297                let install = CanisterInstallError::new(
298                    phase,
299                    None,
300                    Some("fixture".into()),
301                    PocketIcOperationError::new(message),
302                );
303                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
304                let wrapper = WrapperError(std::io::Error::other(install));
305                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
306            }
307        }
308    }
309
310    #[test]
311    fn unrelated_and_quoted_transport_text_does_not_qualify() {
312        for message in [
313            "application worker channel closed",
314            "fixture expected ConnectionRefused but observed another result",
315            "tcp connect error",
316            "connection closed before message completed",
317            "IncompleteMessage",
318            &format!("fixture quoted {REFUSED}"),
319            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
320            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
321            &INCOMPLETE.replace(
322                "hyper::Error(IncompleteMessage)",
323                "Custom(\"channel closed\")",
324            ),
325            &INCOMPLETE.replace(
326                "hyper::Error(IncompleteMessage)",
327                "Custom(\"hyper::Error(IncompleteMessage)\")",
328            ),
329            &format!("fixture quoted {RESET}"),
330            &RESET.replace("/instances/0/", "/application/worker/"),
331            &RESET.replace("/instances/0/", "/instances/quoted/"),
332            &RESET.replace("kind: ConnectionReset,", "kind: ConnectionAborted,"),
333            &RESET.replace("code: 54", "code: quoted"),
334            &RESET.replace("hyper::Error(Io, Os {", "Custom(Os {"),
335        ] {
336            let error = WrapperError(std::io::Error::other(message.to_owned()));
337            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
338            assert!(!PocketIcOperationError::new(message).is_transport());
339            assert!(!panic_is_dead_instance_transport(&message.to_owned()));
340        }
341        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
342        assert!(!is_dead_pocket_ic_transport_error(&refused));
343        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
344        let reset = std::io::Error::from(std::io::ErrorKind::ConnectionReset);
345        assert!(!is_dead_pocket_ic_transport_error(&reset));
346        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(reset)));
347    }
348
349    #[test]
350    fn structured_channel_closure_on_an_instance_request_qualifies() {
351        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
352        assert!(is_dead_instance_transport_error(&message));
353    }
354
355    #[test]
356    fn contextual_call_errors_use_their_structured_kind() {
357        let context = super::super::CandidCallContext::new(
358            "query_call",
359            candid::Principal::anonymous(),
360            candid::Principal::anonymous(),
361            "get",
362        );
363        let error = super::super::CandidCallError::transport(context.clone(), REFUSED);
364        assert!(is_dead_pocket_ic_transport_error(&error));
365        let error = super::super::CandidCallError::decode(context, 0, "channel closed");
366        assert!(!is_dead_pocket_ic_transport_error(&error));
367    }
368
369    #[test]
370    fn io_wrappers_preserve_contextual_call_error_kinds() {
371        let context = super::super::CandidCallContext::new(
372            "query_call",
373            candid::Principal::anonymous(),
374            candid::Principal::anonymous(),
375            "get",
376        );
377        for (error, expected) in [
378            (
379                super::super::CandidCallError::transport(context.clone(), REFUSED),
380                true,
381            ),
382            (
383                super::super::CandidCallError::decode(context, 0, REFUSED),
384                false,
385            ),
386        ] {
387            let inner = std::io::Error::other(error);
388            assert_eq!(is_dead_pocket_ic_transport_error(&inner), expected);
389            let wrapped = WrapperError(std::io::Error::other(inner));
390            assert_eq!(is_dead_pocket_ic_transport_error(&wrapped), expected);
391        }
392    }
393}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
