ic_testkit/pic/
transport.rs

1use std::any::Any;
2
3/// Captured PocketIC operation failure, retaining transport classification.
4#[derive(Clone, Debug, Eq, PartialEq)]
5pub struct PocketIcOperationError {
6    message: String,
7    transport: bool,
8}
9
10impl PocketIcOperationError {
11    /// Capture an upstream PocketIC panic message and classify its transport.
12    #[must_use]
13    pub fn new(message: impl Into<String>) -> Self {
14        let message = message.into();
15        let transport = is_dead_instance_transport_error(&message);
16        Self { message, transport }
17    }
18
19    pub(super) fn from_panic(payload: &(dyn Any + Send)) -> Self {
20        Self::new(panic_payload_to_string(payload))
21    }
22
23    /// Read the original upstream message without contextual wrappers.
24    #[must_use]
25    pub fn message(&self) -> &str {
26        &self.message
27    }
28
29    /// Whether the message matches a recognized PocketIC transport failure.
30    #[must_use]
31    pub const fn is_transport(&self) -> bool {
32        self.transport
33    }
34}
35
36impl std::fmt::Display for PocketIcOperationError {
37    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
38        formatter.write_str(&self.message)
39    }
40}
41
42impl std::error::Error for PocketIcOperationError {}
43
44// Extract a stable string message from one panic payload.
45pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
46    if let Some(message) = payload.downcast_ref::<String>() {
47        return message.clone();
48    }
49    if let Some(message) = payload.downcast_ref::<&'static str>() {
50        return (*message).to_string();
51    }
52
53    "non-string panic payload".to_string()
54}
55
56// Check whether one panic payload belongs to the dead-instance transport class
57// without consuming it, so callers can still resume the original panic.
58pub(super) fn panic_is_dead_instance_transport(payload: &(dyn Any + Send)) -> bool {
59    payload
60        .downcast_ref::<String>()
61        .map(String::as_str)
62        .or_else(|| payload.downcast_ref::<&'static str>().copied())
63        .is_some_and(is_dead_instance_transport_error)
64}
65
66/// Recognize PocketIC transport failures in typed causes and request-error text.
67///
68/// The complete error source chain is inspected so a recipe can classify its
69/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
70/// preserves the classification already captured by testkit's structured errors.
71/// Testkit [`super::CandidCallError`], [`super::CanisterDiagnosticFailure`], and
72/// [`PocketIcOperationError`] causes retain their transport classification through
73/// contextual display wrappers, including nested [`std::io::Error`] values.
74/// Snapshot and installation failures expose their operation cause through
75/// the error source chain.
76///
77/// For other errors, message recognition requires a reqwest debug error with a
78/// PocketIC instance URL and a recognized transport source, optionally prefixed
79/// by PocketIC's HTTP panic context. Generic application messages and bare I/O
80/// error kinds do not qualify.
81///
82/// Use this only for errors originating in a PocketIC operation. This is a
83/// message-based heuristic, not proof that the server or instance has died;
84/// another service or application can reproduce the same text. PocketIC does
85/// not yet expose a structured transport error.
86#[must_use]
87pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
88    let mut current = Some(error);
89    while let Some(candidate) = current {
90        if candidate
91            .downcast_ref::<PocketIcOperationError>()
92            .is_some_and(PocketIcOperationError::is_transport)
93        {
94            return true;
95        }
96        if candidate
97            .downcast_ref::<super::CandidCallError>()
98            .is_some_and(|error| error.kind() == super::CandidCallErrorKind::Transport)
99        {
100            return true;
101        }
102        if matches!(
103            candidate.downcast_ref::<super::CanisterDiagnosticFailure>(),
104            Some(super::CanisterDiagnosticFailure::InstanceUnavailable { .. })
105        ) {
106            return true;
107        }
108        if is_dead_instance_transport_error(&candidate.to_string()) {
109            return true;
110        }
111        // io::Error::source delegates to the contained error's source, skipping
112        // the contained error itself. Inspect it before following its causes so
113        // contextual errors keep their own structured classification.
114        current = if let Some(inner) = candidate
115            .downcast_ref::<std::io::Error>()
116            .and_then(std::io::Error::get_ref)
117        {
118            Some(inner)
119        } else {
120            candidate.source()
121        };
122    }
123    false
124}
125
126// Recognize maintained PocketIC request-error shapes for restore recovery.
127pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
128    let message = message
129        .strip_prefix("HTTP failure: ")
130        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
131        .unwrap_or(message);
132    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
133        return false;
134    };
135    let Some((kind, request)) = error.split_once(", url: \"") else {
136        return false;
137    };
138    if !matches!(kind, "Request" | "Body" | "Decode") {
139        return false;
140    }
141    let Some((url, source)) = request.split_once("\", source: ") else {
142        return false;
143    };
144    let Some(source) = source.strip_suffix(" }") else {
145        return false;
146    };
147    let Some(authority_and_path) = url
148        .strip_prefix("http://")
149        .or_else(|| url.strip_prefix("https://"))
150    else {
151        return false;
152    };
153    let Some((authority, path)) = authority_and_path.split_once('/') else {
154        return false;
155    };
156    let Some(instance_path) = path.strip_prefix("instances/") else {
157        return false;
158    };
159    let instance_id = instance_path.split('/').next().unwrap_or_default();
160    if authority.is_empty()
161        || instance_id.is_empty()
162        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
163    {
164        return false;
165    }
166    let source = source
167        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
168        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
169        .and_then(|source| source.strip_suffix(')'))
170        .unwrap_or(source);
171    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
172        || matches!(
173            source,
174            "hyper::Error(IncompleteMessage)"
175                | "hyper::Error(ChannelClosed)"
176                | "hyper::Error(Canceled, \"connection closed before message completed\")"
177        )
178}
179
180#[cfg(test)]
181mod tests {
182    use super::super::{CanisterInstallError, CanisterInstallPhase, ControllerSnapshotError};
183    use super::{
184        PocketIcOperationError, is_dead_instance_transport_error,
185        is_dead_pocket_ic_transport_error, panic_is_dead_instance_transport,
186    };
187
188    #[derive(Debug)]
189    struct WrapperError(std::io::Error);
190
191    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
192    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
193
194    impl std::fmt::Display for WrapperError {
195        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
196            formatter.write_str("wrapped PocketIC request failed")
197        }
198    }
199
200    impl std::error::Error for WrapperError {
201        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
202            Some(&self.0)
203        }
204    }
205
206    #[test]
207    fn dead_instance_transport_error_detects_connection_refused() {
208        assert!(is_dead_instance_transport_error(REFUSED));
209        assert!(is_dead_instance_transport_error(&format!(
210            "HTTP failure: {REFUSED}"
211        )));
212    }
213
214    #[test]
215    fn dead_instance_transport_error_detects_incomplete_message() {
216        assert!(is_dead_instance_transport_error(INCOMPLETE));
217        assert!(is_dead_instance_transport_error(&format!(
218            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
219        )));
220    }
221
222    #[test]
223    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
224        let classified = PocketIcOperationError::new(INCOMPLETE);
225
226        assert!(classified.is_transport());
227        assert!(panic_is_dead_instance_transport(&INCOMPLETE));
228    }
229
230    #[test]
231    fn public_classifier_inspects_the_error_source_chain() {
232        let dead = WrapperError(std::io::Error::other(REFUSED));
233        assert!(is_dead_pocket_ic_transport_error(&dead));
234
235        let unrelated = WrapperError(std::io::Error::other("request rejected"));
236        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
237    }
238
239    #[test]
240    fn snapshot_and_install_wrappers_preserve_transport_causes() {
241        let canister_id = candid::Principal::anonymous();
242        for message in [REFUSED, "unrelated application panic"] {
243            let expected = message == REFUSED;
244            let capture = ControllerSnapshotError::CapturePanicked {
245                canister_id,
246                source: PocketIcOperationError::new(message),
247                cleanup_failures: vec![],
248            };
249            let restore = ControllerSnapshotError::RestorePanicked {
250                canister_id,
251                source: PocketIcOperationError::new(message),
252            };
253            for snapshot in [capture, restore] {
254                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
255                let wrapper = WrapperError(std::io::Error::other(snapshot));
256                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
257            }
258            for phase in [
259                CanisterInstallPhase::CreateCanister,
260                CanisterInstallPhase::AddCycles,
261                CanisterInstallPhase::InstallCode,
262            ] {
263                let install = CanisterInstallError::new(
264                    phase,
265                    None,
266                    Some("fixture".into()),
267                    PocketIcOperationError::new(message),
268                );
269                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
270                let wrapper = WrapperError(std::io::Error::other(install));
271                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
272            }
273        }
274    }
275
276    #[test]
277    fn unrelated_and_quoted_transport_text_does_not_qualify() {
278        for message in [
279            "application worker channel closed",
280            "fixture expected ConnectionRefused but observed another result",
281            "tcp connect error",
282            "connection closed before message completed",
283            "IncompleteMessage",
284            &format!("fixture quoted {REFUSED}"),
285            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
286            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
287            &INCOMPLETE.replace(
288                "hyper::Error(IncompleteMessage)",
289                "Custom(\"channel closed\")",
290            ),
291            &INCOMPLETE.replace(
292                "hyper::Error(IncompleteMessage)",
293                "Custom(\"hyper::Error(IncompleteMessage)\")",
294            ),
295        ] {
296            let error = WrapperError(std::io::Error::other(message.to_owned()));
297            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
298            assert!(!PocketIcOperationError::new(message).is_transport());
299            assert!(!panic_is_dead_instance_transport(&message.to_owned()));
300        }
301        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
302        assert!(!is_dead_pocket_ic_transport_error(&refused));
303        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
304    }
305
306    #[test]
307    fn structured_channel_closure_on_an_instance_request_qualifies() {
308        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
309        assert!(is_dead_instance_transport_error(&message));
310    }
311
312    #[test]
313    fn contextual_call_errors_use_their_structured_kind() {
314        let context = super::super::CandidCallContext::new(
315            "query_call",
316            candid::Principal::anonymous(),
317            candid::Principal::anonymous(),
318            "get",
319        );
320        let error = super::super::CandidCallError::transport(context.clone(), REFUSED);
321        assert!(is_dead_pocket_ic_transport_error(&error));
322        let error = super::super::CandidCallError::decode(context, 0, "channel closed");
323        assert!(!is_dead_pocket_ic_transport_error(&error));
324    }
325
326    #[test]
327    fn io_wrappers_preserve_contextual_call_error_kinds() {
328        let context = super::super::CandidCallContext::new(
329            "query_call",
330            candid::Principal::anonymous(),
331            candid::Principal::anonymous(),
332            "get",
333        );
334        for (error, expected) in [
335            (
336                super::super::CandidCallError::transport(context.clone(), REFUSED),
337                true,
338            ),
339            (
340                super::super::CandidCallError::decode(context, 0, REFUSED),
341                false,
342            ),
343        ] {
344            let inner = std::io::Error::other(error);
345            assert_eq!(is_dead_pocket_ic_transport_error(&inner), expected);
346            let wrapped = WrapperError(std::io::Error::other(inner));
347            assert_eq!(is_dead_pocket_ic_transport_error(&wrapped), expected);
348        }
349    }
350}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
