ic_testkit/pic/
transport.rs

1use std::{
2    any::Any,
3    panic::{AssertUnwindSafe, catch_unwind},
4};
5
6use pocket_ic::RejectResponse;
7
8/// Captured PocketIC operation failure, retaining transport classification.
9#[derive(Clone, Debug, Eq, PartialEq)]
10pub struct PocketIcOperationError {
11    message: String,
12    transport: bool,
13}
14
15impl PocketIcOperationError {
16    /// Capture an upstream PocketIC panic message and classify its transport.
17    #[must_use]
18    pub fn new(message: impl Into<String>) -> Self {
19        let message = message.into();
20        let transport = is_dead_instance_transport_error(&message);
21        Self { message, transport }
22    }
23
24    /// Read the original upstream message without contextual wrappers.
25    #[must_use]
26    pub fn message(&self) -> &str {
27        &self.message
28    }
29
30    /// Whether the message matches a recognized PocketIC transport failure.
31    #[must_use]
32    pub const fn is_transport(&self) -> bool {
33        self.transport
34    }
35}
36
37// Only callers that intentionally capture every unwind use this boundary.
38// Candid calls retain their separate policy of resuming unrelated panics.
39pub(super) fn capture_operation<T>(
40    operation: impl FnOnce() -> T,
41) -> Result<T, PocketIcOperationError> {
42    catch_unwind(AssertUnwindSafe(operation))
43        .map_err(|payload| PocketIcOperationError::new(panic_payload_to_string(payload.as_ref())))
44}
45
46impl std::fmt::Display for PocketIcOperationError {
47    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48        formatter.write_str(&self.message)
49    }
50}
51
52impl std::error::Error for PocketIcOperationError {}
53
54/// A rejected management call or a captured PocketIC operation panic.
55#[non_exhaustive]
56#[derive(Debug)]
57pub enum PocketIcCallFailure {
58    /// PocketIC returned a structured management-canister rejection.
59    Rejected(RejectResponse),
60    /// Captured PocketIC panic, retaining its original transport classification.
61    Operation(PocketIcOperationError),
62}
63
64impl std::fmt::Display for PocketIcCallFailure {
65    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66        match self {
67            Self::Rejected(response) => write!(formatter, "rejected: {response:?}"),
68            Self::Operation(source) if source.is_transport() => {
69                write!(formatter, "PocketIC instance unavailable: {source}")
70            }
71            Self::Operation(source) => write!(formatter, "panicked: {source}"),
72        }
73    }
74}
75
76impl std::error::Error for PocketIcCallFailure {
77    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
78        match self {
79            Self::Rejected(_) => None,
80            Self::Operation(source) => Some(source),
81        }
82    }
83}
84
85pub(super) fn capture_call<T>(
86    call: impl FnOnce() -> Result<T, RejectResponse>,
87) -> Result<T, PocketIcCallFailure> {
88    match capture_operation(call) {
89        Ok(Ok(value)) => Ok(value),
90        Ok(Err(response)) => Err(PocketIcCallFailure::Rejected(response)),
91        Err(source) => Err(PocketIcCallFailure::Operation(source)),
92    }
93}
94
95// Extract a stable string message from one panic payload.
96pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
97    if let Some(message) = payload.downcast_ref::<String>() {
98        return message.clone();
99    }
100    if let Some(message) = payload.downcast_ref::<&'static str>() {
101        return (*message).to_string();
102    }
103
104    "non-string panic payload".to_string()
105}
106
107/// Recognize PocketIC transport failures in typed causes and request-error text.
108///
109/// The complete error source chain is inspected so a recipe can classify its
110/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
111/// preserves the classification already captured by testkit's structured errors.
112/// Testkit [`super::CandidCallError`], [`super::PocketIcCallFailure`], and
113/// [`PocketIcOperationError`] causes retain their transport classification through
114/// contextual display wrappers, including nested [`std::io::Error`] values.
115/// Snapshot and installation failures expose their operation cause through
116/// the error source chain.
117///
118/// For other errors, message recognition requires a reqwest debug error with a
119/// PocketIC instance URL and a recognized transport source, optionally prefixed
120/// by PocketIC's HTTP panic context. Generic application messages and bare I/O
121/// error kinds do not qualify.
122/// Recognized sources include OS connection resets during an HTTP request.
123///
124/// Use this only for errors originating in a PocketIC operation. This is a
125/// message-based heuristic, not proof that the server or instance has died;
126/// another service or application can reproduce the same text. PocketIC does
127/// not yet expose a structured transport error.
128#[must_use]
129pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
130    let mut current = Some(error);
131    while let Some(candidate) = current {
132        // Decoder diagnostics may quote arbitrary reply content. Reaching a
133        // typed decode failure proves this branch is not a transport failure.
134        if candidate.is::<ic_testkit_candid::DecodeError>() {
135            return false;
136        }
137        if candidate
138            .downcast_ref::<PocketIcOperationError>()
139            .is_some_and(PocketIcOperationError::is_transport)
140        {
141            return true;
142        }
143        if is_dead_instance_transport_error(&candidate.to_string()) {
144            return true;
145        }
146        // io::Error::source delegates to the contained error's source, skipping
147        // the contained error itself. Inspect it before following its causes so
148        // contextual errors keep their own structured classification.
149        current = if let Some(inner) = candidate
150            .downcast_ref::<std::io::Error>()
151            .and_then(std::io::Error::get_ref)
152        {
153            Some(inner)
154        } else {
155            candidate.source()
156        };
157    }
158    false
159}
160
161// Recognize maintained PocketIC request-error shapes for restore recovery.
162pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
163    let message = message
164        .strip_prefix("HTTP failure: ")
165        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
166        .unwrap_or(message);
167    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
168        return false;
169    };
170    let Some((kind, request)) = error.split_once(", url: \"") else {
171        return false;
172    };
173    if !matches!(kind, "Request" | "Body" | "Decode") {
174        return false;
175    }
176    let Some((url, source)) = request.split_once("\", source: ") else {
177        return false;
178    };
179    let Some(source) = source.strip_suffix(" }") else {
180        return false;
181    };
182    let Some(authority_and_path) = url
183        .strip_prefix("http://")
184        .or_else(|| url.strip_prefix("https://"))
185    else {
186        return false;
187    };
188    let Some((authority, path)) = authority_and_path.split_once('/') else {
189        return false;
190    };
191    let Some(instance_path) = path.strip_prefix("instances/") else {
192        return false;
193    };
194    let instance_id = instance_path.split('/').next().unwrap_or_default();
195    if authority.is_empty()
196        || instance_id.is_empty()
197        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
198    {
199        return false;
200    }
201    let source = source
202        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
203        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
204        .and_then(|source| source.strip_suffix(')'))
205        .unwrap_or(source);
206    let reset = source
207        .strip_prefix("hyper::Error(Io, Os { code: ")
208        .and_then(|source| source.split_once(", kind: ConnectionReset, message: \""))
209        .is_some_and(|(code, message)| code.parse::<i32>().is_ok() && message.ends_with("\" })"));
210    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
211        || reset
212        || matches!(
213            source,
214            "hyper::Error(IncompleteMessage)"
215                | "hyper::Error(ChannelClosed)"
216                | "hyper::Error(Canceled, \"connection closed before message completed\")"
217        )
218}
219
220#[cfg(test)]
221mod tests {
222    use super::super::{CanisterInstallError, ControllerSnapshotError};
223    use super::{
224        PocketIcCallFailure, PocketIcOperationError, capture_operation,
225        is_dead_instance_transport_error, is_dead_pocket_ic_transport_error,
226    };
227
228    #[derive(Debug)]
229    struct WrapperError(std::io::Error);
230
231    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
232    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
233    const RESET: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:49238/instances/0/update/submit_ingress_message\", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(Io, Os { code: 54, kind: ConnectionReset, message: \"Connection reset by peer\" })) }";
234
235    impl std::fmt::Display for WrapperError {
236        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
237            formatter.write_str("wrapped PocketIC request failed")
238        }
239    }
240
241    impl std::error::Error for WrapperError {
242        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
243            Some(&self.0)
244        }
245    }
246
247    #[test]
248    fn dead_instance_transport_error_detects_connection_refused() {
249        assert!(is_dead_instance_transport_error(REFUSED));
250        assert!(is_dead_instance_transport_error(&format!(
251            "HTTP failure: {REFUSED}"
252        )));
253    }
254
255    #[test]
256    fn dead_instance_transport_error_detects_incomplete_message() {
257        assert!(is_dead_instance_transport_error(INCOMPLETE));
258        assert!(is_dead_instance_transport_error(&format!(
259            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
260        )));
261    }
262
263    #[test]
264    fn dead_instance_transport_error_detects_native_connection_reset() {
265        for message in [RESET.to_owned(), RESET.replace("code: 54", "code: 104")] {
266            assert!(is_dead_instance_transport_error(&message));
267            assert!(PocketIcOperationError::new(format!("HTTP failure: {message}")).is_transport());
268            assert!(is_dead_instance_transport_error(&format!(
269                "called `Result::unwrap()` on an `Err` value: {message}"
270            )));
271        }
272    }
273
274    #[test]
275    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
276        let classified = capture_operation::<()>(|| panic!("{INCOMPLETE}"))
277            .expect_err("operation panic must be captured");
278
279        assert_eq!(classified.message(), INCOMPLETE);
280        assert!(classified.is_transport());
281        let non_string = capture_operation::<()>(|| std::panic::panic_any(7_u8))
282            .expect_err("non-string panic must be captured");
283        assert_eq!(non_string.message(), "non-string panic payload");
284        assert!(!non_string.is_transport());
285    }
286
287    #[test]
288    fn public_classifier_inspects_the_error_source_chain() {
289        let dead = WrapperError(std::io::Error::other(REFUSED));
290        assert!(is_dead_pocket_ic_transport_error(&dead));
291
292        let unrelated = WrapperError(std::io::Error::other("request rejected"));
293        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
294    }
295
296    #[test]
297    fn snapshot_and_install_wrappers_preserve_transport_causes() {
298        let canister_id = candid::Principal::anonymous();
299        for message in [REFUSED, RESET, "unrelated application panic"] {
300            let expected = message != "unrelated application panic";
301            let capture = ControllerSnapshotError::CaptureFailed {
302                canister_id,
303                sender: None,
304                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
305                    message,
306                ))),
307                cleanup_failures: vec![],
308            };
309            let restore = ControllerSnapshotError::RestoreFailed {
310                canister_id,
311                sender: None,
312                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
313                    message,
314                ))),
315            };
316            for snapshot in [capture, restore] {
317                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
318                let wrapper = WrapperError(std::io::Error::other(snapshot));
319                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
320            }
321            for install in [
322                CanisterInstallError::CreateCanister {
323                    label: Some("fixture".into()),
324                    source: PocketIcOperationError::new(message),
325                },
326                CanisterInstallError::AddCycles {
327                    canister_id,
328                    label: Some("fixture".into()),
329                    source: PocketIcOperationError::new(message),
330                },
331                CanisterInstallError::InstallCode {
332                    canister_id,
333                    label: Some("fixture".into()),
334                    source: PocketIcOperationError::new(message),
335                },
336            ] {
337                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
338                let wrapper = WrapperError(std::io::Error::other(install));
339                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
340            }
341        }
342    }
343
344    #[test]
345    fn unrelated_and_quoted_transport_text_does_not_qualify() {
346        for message in [
347            "application worker channel closed",
348            "fixture expected ConnectionRefused but observed another result",
349            "tcp connect error",
350            "connection closed before message completed",
351            "IncompleteMessage",
352            &format!("fixture quoted {REFUSED}"),
353            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
354            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
355            &INCOMPLETE.replace(
356                "hyper::Error(IncompleteMessage)",
357                "Custom(\"channel closed\")",
358            ),
359            &INCOMPLETE.replace(
360                "hyper::Error(IncompleteMessage)",
361                "Custom(\"hyper::Error(IncompleteMessage)\")",
362            ),
363            &format!("fixture quoted {RESET}"),
364            &RESET.replace("/instances/0/", "/application/worker/"),
365            &RESET.replace("/instances/0/", "/instances/quoted/"),
366            &RESET.replace("kind: ConnectionReset,", "kind: ConnectionAborted,"),
367            &RESET.replace("code: 54", "code: quoted"),
368            &RESET.replace("hyper::Error(Io, Os {", "Custom(Os {"),
369        ] {
370            let error = WrapperError(std::io::Error::other(message.to_owned()));
371            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
372            assert!(!PocketIcOperationError::new(message).is_transport());
373        }
374        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
375        assert!(!is_dead_pocket_ic_transport_error(&refused));
376        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
377        let reset = std::io::Error::from(std::io::ErrorKind::ConnectionReset);
378        assert!(!is_dead_pocket_ic_transport_error(&reset));
379        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(reset)));
380    }
381
382    #[test]
383    fn structured_channel_closure_on_an_instance_request_qualifies() {
384        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
385        assert!(is_dead_instance_transport_error(&message));
386    }
387
388    #[test]
389    fn contextual_call_errors_use_their_structured_kind() {
390        let context = super::super::CandidCallContext::new(
391            "query_call",
392            candid::Principal::anonymous(),
393            candid::Principal::anonymous(),
394            "get",
395        );
396        let error = super::super::CandidCallError::Transport {
397            context: Box::new(context.clone()),
398            source: PocketIcOperationError::new(REFUSED),
399        };
400        assert!(is_dead_pocket_ic_transport_error(&error));
401        let error = super::super::CandidCallError::Decode {
402            context: Box::new(context),
403            bytes: 0,
404            source: Box::new(ic_testkit_candid::DecodeError::Candid(candid::Error::msg(
405                "channel closed",
406            ))),
407        };
408        assert!(!is_dead_pocket_ic_transport_error(&error));
409    }
410
411    #[test]
412    fn io_wrappers_preserve_contextual_call_error_kinds() {
413        let context = super::super::CandidCallContext::new(
414            "query_call",
415            candid::Principal::anonymous(),
416            candid::Principal::anonymous(),
417            "get",
418        );
419        for (error, expected) in [
420            (
421                super::super::CandidCallError::Transport {
422                    context: Box::new(context.clone()),
423                    source: PocketIcOperationError::new(REFUSED),
424                },
425                true,
426            ),
427            (
428                super::super::CandidCallError::Decode {
429                    context: Box::new(context),
430                    bytes: 0,
431                    source: Box::new(ic_testkit_candid::DecodeError::Candid(candid::Error::msg(
432                        REFUSED,
433                    ))),
434                },
435                false,
436            ),
437        ] {
438            let inner = std::io::Error::other(error);
439            assert_eq!(is_dead_pocket_ic_transport_error(&inner), expected);
440            let wrapped = WrapperError(std::io::Error::other(inner));
441            assert_eq!(is_dead_pocket_ic_transport_error(&wrapped), expected);
442        }
443    }
444}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
