ic_testkit/pic/
transport.rs

1use std::any::Any;
2
3/// Captured PocketIC operation failure, retaining transport classification.
4#[derive(Clone, Debug, Eq, PartialEq)]
5pub struct PocketIcOperationError {
6    message: String,
7    transport: bool,
8}
9
10impl PocketIcOperationError {
11    /// Capture an upstream PocketIC panic message and classify its transport.
12    #[must_use]
13    pub fn new(message: impl Into<String>) -> Self {
14        let message = message.into();
15        let transport = is_dead_instance_transport_error(&message);
16        Self { message, transport }
17    }
18
19    pub(super) fn from_panic(payload: &(dyn Any + Send)) -> Self {
20        Self::new(panic_payload_to_string(payload))
21    }
22
23    /// Read the original upstream message without contextual wrappers.
24    #[must_use]
25    pub fn message(&self) -> &str {
26        &self.message
27    }
28
29    /// Whether the message matches a recognized PocketIC transport failure.
30    #[must_use]
31    pub const fn is_transport(&self) -> bool {
32        self.transport
33    }
34}
35
36impl std::fmt::Display for PocketIcOperationError {
37    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
38        formatter.write_str(&self.message)
39    }
40}
41
42impl std::error::Error for PocketIcOperationError {}
43
44// Extract a stable string message from one panic payload.
45pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
46    if let Some(message) = payload.downcast_ref::<String>() {
47        return message.clone();
48    }
49    if let Some(message) = payload.downcast_ref::<&'static str>() {
50        return (*message).to_string();
51    }
52
53    "non-string panic payload".to_string()
54}
55
56// Check whether one panic payload belongs to the dead-instance transport class
57// without consuming it, so callers can still resume the original panic.
58pub(super) fn panic_is_dead_instance_transport(payload: &(dyn Any + Send)) -> bool {
59    payload
60        .downcast_ref::<String>()
61        .map(String::as_str)
62        .or_else(|| payload.downcast_ref::<&'static str>().copied())
63        .is_some_and(is_dead_instance_transport_error)
64}
65
66/// Recognize PocketIC transport failures in typed causes and request-error text.
67///
68/// The complete error source chain is inspected so a recipe can classify its
69/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
70/// preserves the classification already captured by testkit's structured errors.
71/// Testkit [`super::CandidCallError`], [`super::CanisterDiagnosticFailure`], and
72/// [`PocketIcOperationError`] causes retain their transport classification through
73/// contextual display wrappers, including nested [`std::io::Error`] values.
74/// Snapshot and installation failures expose their operation cause through
75/// the error source chain.
76///
77/// For other errors, message recognition requires a reqwest debug error with a
78/// PocketIC instance URL and a recognized transport source, optionally prefixed
79/// by PocketIC's HTTP panic context. Generic application messages and bare I/O
80/// error kinds do not qualify.
81/// Recognized sources include OS connection resets during an HTTP request.
82///
83/// Use this only for errors originating in a PocketIC operation. This is a
84/// message-based heuristic, not proof that the server or instance has died;
85/// another service or application can reproduce the same text. PocketIC does
86/// not yet expose a structured transport error.
87#[must_use]
88pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
89    let mut current = Some(error);
90    while let Some(candidate) = current {
91        if candidate
92            .downcast_ref::<PocketIcOperationError>()
93            .is_some_and(PocketIcOperationError::is_transport)
94        {
95            return true;
96        }
97        if candidate
98            .downcast_ref::<super::CandidCallError>()
99            .is_some_and(|error| error.kind() == super::CandidCallErrorKind::Transport)
100        {
101            return true;
102        }
103        if matches!(
104            candidate.downcast_ref::<super::CanisterDiagnosticFailure>(),
105            Some(super::CanisterDiagnosticFailure::InstanceUnavailable { .. })
106        ) {
107            return true;
108        }
109        if is_dead_instance_transport_error(&candidate.to_string()) {
110            return true;
111        }
112        // io::Error::source delegates to the contained error's source, skipping
113        // the contained error itself. Inspect it before following its causes so
114        // contextual errors keep their own structured classification.
115        current = if let Some(inner) = candidate
116            .downcast_ref::<std::io::Error>()
117            .and_then(std::io::Error::get_ref)
118        {
119            Some(inner)
120        } else {
121            candidate.source()
122        };
123    }
124    false
125}
126
127// Recognize maintained PocketIC request-error shapes for restore recovery.
128pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
129    let message = message
130        .strip_prefix("HTTP failure: ")
131        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
132        .unwrap_or(message);
133    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
134        return false;
135    };
136    let Some((kind, request)) = error.split_once(", url: \"") else {
137        return false;
138    };
139    if !matches!(kind, "Request" | "Body" | "Decode") {
140        return false;
141    }
142    let Some((url, source)) = request.split_once("\", source: ") else {
143        return false;
144    };
145    let Some(source) = source.strip_suffix(" }") else {
146        return false;
147    };
148    let Some(authority_and_path) = url
149        .strip_prefix("http://")
150        .or_else(|| url.strip_prefix("https://"))
151    else {
152        return false;
153    };
154    let Some((authority, path)) = authority_and_path.split_once('/') else {
155        return false;
156    };
157    let Some(instance_path) = path.strip_prefix("instances/") else {
158        return false;
159    };
160    let instance_id = instance_path.split('/').next().unwrap_or_default();
161    if authority.is_empty()
162        || instance_id.is_empty()
163        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
164    {
165        return false;
166    }
167    let source = source
168        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
169        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
170        .and_then(|source| source.strip_suffix(')'))
171        .unwrap_or(source);
172    let reset = source
173        .strip_prefix("hyper::Error(Io, Os { code: ")
174        .and_then(|source| source.split_once(", kind: ConnectionReset, message: \""))
175        .is_some_and(|(code, message)| code.parse::<i32>().is_ok() && message.ends_with("\" })"));
176    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
177        || reset
178        || matches!(
179            source,
180            "hyper::Error(IncompleteMessage)"
181                | "hyper::Error(ChannelClosed)"
182                | "hyper::Error(Canceled, \"connection closed before message completed\")"
183        )
184}
185
186#[cfg(test)]
187mod tests {
188    use super::super::{CanisterInstallError, CanisterInstallPhase, ControllerSnapshotError};
189    use super::{
190        PocketIcOperationError, is_dead_instance_transport_error,
191        is_dead_pocket_ic_transport_error, panic_is_dead_instance_transport,
192    };
193
194    #[derive(Debug)]
195    struct WrapperError(std::io::Error);
196
197    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
198    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
199    const RESET: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:49238/instances/0/update/submit_ingress_message\", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(Io, Os { code: 54, kind: ConnectionReset, message: \"Connection reset by peer\" })) }";
200
201    impl std::fmt::Display for WrapperError {
202        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
203            formatter.write_str("wrapped PocketIC request failed")
204        }
205    }
206
207    impl std::error::Error for WrapperError {
208        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
209            Some(&self.0)
210        }
211    }
212
213    #[test]
214    fn dead_instance_transport_error_detects_connection_refused() {
215        assert!(is_dead_instance_transport_error(REFUSED));
216        assert!(is_dead_instance_transport_error(&format!(
217            "HTTP failure: {REFUSED}"
218        )));
219    }
220
221    #[test]
222    fn dead_instance_transport_error_detects_incomplete_message() {
223        assert!(is_dead_instance_transport_error(INCOMPLETE));
224        assert!(is_dead_instance_transport_error(&format!(
225            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
226        )));
227    }
228
229    #[test]
230    fn dead_instance_transport_error_detects_native_connection_reset() {
231        for message in [RESET.to_owned(), RESET.replace("code: 54", "code: 104")] {
232            assert!(is_dead_instance_transport_error(&message));
233            assert!(panic_is_dead_instance_transport(&format!(
234                "HTTP failure: {message}"
235            )));
236            assert!(is_dead_instance_transport_error(&format!(
237                "called `Result::unwrap()` on an `Err` value: {message}"
238            )));
239        }
240    }
241
242    #[test]
243    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
244        let classified = PocketIcOperationError::new(INCOMPLETE);
245
246        assert!(classified.is_transport());
247        assert!(panic_is_dead_instance_transport(&INCOMPLETE));
248    }
249
250    #[test]
251    fn public_classifier_inspects_the_error_source_chain() {
252        let dead = WrapperError(std::io::Error::other(REFUSED));
253        assert!(is_dead_pocket_ic_transport_error(&dead));
254
255        let unrelated = WrapperError(std::io::Error::other("request rejected"));
256        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
257    }
258
259    #[test]
260    fn snapshot_and_install_wrappers_preserve_transport_causes() {
261        let canister_id = candid::Principal::anonymous();
262        for message in [REFUSED, RESET, "unrelated application panic"] {
263            let expected = message != "unrelated application panic";
264            let capture = ControllerSnapshotError::CapturePanicked {
265                canister_id,
266                source: PocketIcOperationError::new(message),
267                cleanup_failures: vec![],
268            };
269            let restore = ControllerSnapshotError::RestorePanicked {
270                canister_id,
271                source: PocketIcOperationError::new(message),
272            };
273            for snapshot in [capture, restore] {
274                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
275                let wrapper = WrapperError(std::io::Error::other(snapshot));
276                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
277            }
278            for phase in [
279                CanisterInstallPhase::CreateCanister,
280                CanisterInstallPhase::AddCycles,
281                CanisterInstallPhase::InstallCode,
282            ] {
283                let install = CanisterInstallError::new(
284                    phase,
285                    None,
286                    Some("fixture".into()),
287                    PocketIcOperationError::new(message),
288                );
289                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
290                let wrapper = WrapperError(std::io::Error::other(install));
291                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
292            }
293        }
294    }
295
296    #[test]
297    fn unrelated_and_quoted_transport_text_does_not_qualify() {
298        for message in [
299            "application worker channel closed",
300            "fixture expected ConnectionRefused but observed another result",
301            "tcp connect error",
302            "connection closed before message completed",
303            "IncompleteMessage",
304            &format!("fixture quoted {REFUSED}"),
305            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
306            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
307            &INCOMPLETE.replace(
308                "hyper::Error(IncompleteMessage)",
309                "Custom(\"channel closed\")",
310            ),
311            &INCOMPLETE.replace(
312                "hyper::Error(IncompleteMessage)",
313                "Custom(\"hyper::Error(IncompleteMessage)\")",
314            ),
315            &format!("fixture quoted {RESET}"),
316            &RESET.replace("/instances/0/", "/application/worker/"),
317            &RESET.replace("/instances/0/", "/instances/quoted/"),
318            &RESET.replace("kind: ConnectionReset,", "kind: ConnectionAborted,"),
319            &RESET.replace("code: 54", "code: quoted"),
320            &RESET.replace("hyper::Error(Io, Os {", "Custom(Os {"),
321        ] {
322            let error = WrapperError(std::io::Error::other(message.to_owned()));
323            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
324            assert!(!PocketIcOperationError::new(message).is_transport());
325            assert!(!panic_is_dead_instance_transport(&message.to_owned()));
326        }
327        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
328        assert!(!is_dead_pocket_ic_transport_error(&refused));
329        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
330        let reset = std::io::Error::from(std::io::ErrorKind::ConnectionReset);
331        assert!(!is_dead_pocket_ic_transport_error(&reset));
332        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(reset)));
333    }
334
335    #[test]
336    fn structured_channel_closure_on_an_instance_request_qualifies() {
337        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
338        assert!(is_dead_instance_transport_error(&message));
339    }
340
341    #[test]
342    fn contextual_call_errors_use_their_structured_kind() {
343        let context = super::super::CandidCallContext::new(
344            "query_call",
345            candid::Principal::anonymous(),
346            candid::Principal::anonymous(),
347            "get",
348        );
349        let error = super::super::CandidCallError::transport(context.clone(), REFUSED);
350        assert!(is_dead_pocket_ic_transport_error(&error));
351        let error = super::super::CandidCallError::decode(context, 0, "channel closed");
352        assert!(!is_dead_pocket_ic_transport_error(&error));
353    }
354
355    #[test]
356    fn io_wrappers_preserve_contextual_call_error_kinds() {
357        let context = super::super::CandidCallContext::new(
358            "query_call",
359            candid::Principal::anonymous(),
360            candid::Principal::anonymous(),
361            "get",
362        );
363        for (error, expected) in [
364            (
365                super::super::CandidCallError::transport(context.clone(), REFUSED),
366                true,
367            ),
368            (
369                super::super::CandidCallError::decode(context, 0, REFUSED),
370                false,
371            ),
372        ] {
373            let inner = std::io::Error::other(error);
374            assert_eq!(is_dead_pocket_ic_transport_error(&inner), expected);
375            let wrapped = WrapperError(std::io::Error::other(inner));
376            assert_eq!(is_dead_pocket_ic_transport_error(&wrapped), expected);
377        }
378    }
379}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
