ic_testkit/pic/
transport.rs

1use std::any::Any;
2
3/// Captured PocketIC operation failure, retaining transport classification.
4#[derive(Clone, Debug, Eq, PartialEq)]
5pub struct PocketIcOperationError {
6    message: String,
7    transport: bool,
8}
9
10impl PocketIcOperationError {
11    /// Capture an upstream PocketIC panic message and classify its transport.
12    #[must_use]
13    pub fn new(message: impl Into<String>) -> Self {
14        let message = message.into();
15        let transport = message_is_dead_instance_transport_error(&message);
16        Self { message, transport }
17    }
18
19    pub(super) fn from_panic(payload: &(dyn Any + Send)) -> Self {
20        Self::new(panic_payload_to_string(payload))
21    }
22
23    /// Read the original upstream message without contextual wrappers.
24    #[must_use]
25    pub fn message(&self) -> &str {
26        &self.message
27    }
28
29    /// Whether the message matches a recognized PocketIC transport failure.
30    #[must_use]
31    pub const fn is_transport(&self) -> bool {
32        self.transport
33    }
34}
35
36impl std::fmt::Display for PocketIcOperationError {
37    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
38        formatter.write_str(&self.message)
39    }
40}
41
42impl std::error::Error for PocketIcOperationError {}
43
44// Extract a stable string message from one panic payload.
45pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
46    if let Some(message) = payload.downcast_ref::<String>() {
47        return message.clone();
48    }
49    if let Some(message) = payload.downcast_ref::<&'static str>() {
50        return (*message).to_string();
51    }
52
53    "non-string panic payload".to_string()
54}
55
56// Check whether one panic payload belongs to the dead-instance transport class
57// without consuming it, so callers can still resume the original panic.
58pub(super) fn panic_is_dead_instance_transport(payload: &(dyn Any + Send)) -> bool {
59    payload
60        .downcast_ref::<String>()
61        .map(String::as_str)
62        .or_else(|| payload.downcast_ref::<&'static str>().copied())
63        .is_some_and(message_is_dead_instance_transport_error)
64}
65
66// Recognize maintained PocketIC request-error shapes for restore recovery.
67pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
68    message_is_dead_instance_transport_error(message)
69}
70
71/// Recognize transport failures in PocketIC's unstructured request errors.
72///
73/// The complete error source chain is inspected so a recipe can classify its
74/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
75/// requires a reqwest debug error with a PocketIC instance URL and a recognized
76/// transport source, optionally prefixed by PocketIC's HTTP panic context.
77/// Generic application messages and bare I/O error kinds do not qualify.
78/// Testkit [`super::CandidCallError`] and [`PocketIcOperationError`] causes
79/// retain their transport classification through contextual display wrappers.
80/// Snapshot and installation failures expose their operation cause through
81/// the error source chain.
82///
83/// Use this only for errors originating in a PocketIC operation. This is a
84/// message-based heuristic, not proof that the server or instance has died;
85/// another service or application can reproduce the same text. PocketIC does
86/// not yet expose a structured transport error.
87#[must_use]
88pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
89    let mut current = Some(error);
90    while let Some(candidate) = current {
91        if candidate
92            .downcast_ref::<PocketIcOperationError>()
93            .is_some_and(PocketIcOperationError::is_transport)
94        {
95            return true;
96        }
97        if candidate
98            .downcast_ref::<super::CandidCallError>()
99            .is_some_and(|error| error.kind() == super::CandidCallErrorKind::Transport)
100        {
101            return true;
102        }
103        if message_is_dead_instance_transport_error(&candidate.to_string()) {
104            return true;
105        }
106        current = candidate.source();
107    }
108    false
109}
110
111fn message_is_dead_instance_transport_error(message: &str) -> bool {
112    let message = message
113        .strip_prefix("HTTP failure: ")
114        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
115        .unwrap_or(message);
116    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
117        return false;
118    };
119    let Some((kind, request)) = error.split_once(", url: \"") else {
120        return false;
121    };
122    if !matches!(kind, "Request" | "Body" | "Decode") {
123        return false;
124    }
125    let Some((url, source)) = request.split_once("\", source: ") else {
126        return false;
127    };
128    let Some(source) = source.strip_suffix(" }") else {
129        return false;
130    };
131    let Some(authority_and_path) = url
132        .strip_prefix("http://")
133        .or_else(|| url.strip_prefix("https://"))
134    else {
135        return false;
136    };
137    let Some((authority, path)) = authority_and_path.split_once('/') else {
138        return false;
139    };
140    let Some(instance_path) = path.strip_prefix("instances/") else {
141        return false;
142    };
143    let instance_id = instance_path.split('/').next().unwrap_or_default();
144    if authority.is_empty()
145        || instance_id.is_empty()
146        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
147    {
148        return false;
149    }
150    let source = source
151        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
152        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
153        .and_then(|source| source.strip_suffix(')'))
154        .unwrap_or(source);
155    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
156        || matches!(
157            source,
158            "hyper::Error(IncompleteMessage)"
159                | "hyper::Error(ChannelClosed)"
160                | "hyper::Error(Canceled, \"connection closed before message completed\")"
161        )
162}
163
164#[cfg(test)]
165mod tests {
166    use super::{
167        PocketIcOperationError, is_dead_instance_transport_error,
168        is_dead_pocket_ic_transport_error, panic_is_dead_instance_transport,
169    };
170
171    #[derive(Debug)]
172    struct WrapperError(std::io::Error);
173
174    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
175    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
176
177    impl std::fmt::Display for WrapperError {
178        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
179            formatter.write_str("wrapped PocketIC request failed")
180        }
181    }
182
183    impl std::error::Error for WrapperError {
184        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
185            Some(&self.0)
186        }
187    }
188
189    #[test]
190    fn dead_instance_transport_error_detects_connection_refused() {
191        assert!(is_dead_instance_transport_error(REFUSED));
192        assert!(is_dead_instance_transport_error(&format!(
193            "HTTP failure: {REFUSED}"
194        )));
195    }
196
197    #[test]
198    fn dead_instance_transport_error_detects_incomplete_message() {
199        assert!(is_dead_instance_transport_error(INCOMPLETE));
200        assert!(is_dead_instance_transport_error(&format!(
201            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
202        )));
203    }
204
205    #[test]
206    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
207        let classified = PocketIcOperationError::new(INCOMPLETE);
208
209        assert!(classified.is_transport());
210        assert!(panic_is_dead_instance_transport(&INCOMPLETE));
211    }
212
213    #[test]
214    fn public_classifier_inspects_the_error_source_chain() {
215        let dead = WrapperError(std::io::Error::other(REFUSED));
216        assert!(is_dead_pocket_ic_transport_error(&dead));
217
218        let unrelated = WrapperError(std::io::Error::other("request rejected"));
219        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
220    }
221
222    #[test]
223    fn snapshot_and_install_wrappers_preserve_transport_causes() {
224        use super::super::{CanisterInstallError, CanisterInstallPhase, ControllerSnapshotError};
225        let canister_id = candid::Principal::anonymous();
226        for message in [REFUSED, "unrelated application panic"] {
227            let expected = message == REFUSED;
228            let capture = ControllerSnapshotError::CapturePanicked {
229                canister_id,
230                source: PocketIcOperationError::new(message),
231                cleanup_failures: vec![],
232            };
233            let restore = ControllerSnapshotError::RestorePanicked {
234                canister_id,
235                source: PocketIcOperationError::new(message),
236            };
237            for snapshot in [capture, restore] {
238                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
239                let wrapper = WrapperError(std::io::Error::other(snapshot));
240                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
241            }
242            for phase in [
243                CanisterInstallPhase::CreateCanister,
244                CanisterInstallPhase::AddCycles,
245                CanisterInstallPhase::InstallCode,
246            ] {
247                let install = CanisterInstallError::new(
248                    phase,
249                    None,
250                    Some("fixture".into()),
251                    PocketIcOperationError::new(message),
252                );
253                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
254                let wrapper = WrapperError(std::io::Error::other(install));
255                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
256            }
257        }
258    }
259
260    #[test]
261    fn unrelated_and_quoted_transport_text_does_not_qualify() {
262        for message in [
263            "application worker channel closed",
264            "fixture expected ConnectionRefused but observed another result",
265            "tcp connect error",
266            "connection closed before message completed",
267            "IncompleteMessage",
268            &format!("fixture quoted {REFUSED}"),
269            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
270            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
271            &INCOMPLETE.replace(
272                "hyper::Error(IncompleteMessage)",
273                "Custom(\"channel closed\")",
274            ),
275            &INCOMPLETE.replace(
276                "hyper::Error(IncompleteMessage)",
277                "Custom(\"hyper::Error(IncompleteMessage)\")",
278            ),
279        ] {
280            let error = WrapperError(std::io::Error::other(message.to_owned()));
281            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
282            assert!(!PocketIcOperationError::new(message).is_transport());
283            assert!(!panic_is_dead_instance_transport(&message.to_owned()));
284        }
285        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
286        assert!(!is_dead_pocket_ic_transport_error(&refused));
287        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
288    }
289
290    #[test]
291    fn structured_channel_closure_on_an_instance_request_qualifies() {
292        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
293        assert!(is_dead_instance_transport_error(&message));
294    }
295
296    #[test]
297    fn contextual_call_errors_use_their_structured_kind() {
298        let context = super::super::CandidCallContext::new(
299            "query_call",
300            candid::Principal::anonymous(),
301            candid::Principal::anonymous(),
302            "get",
303        );
304        let error = super::super::CandidCallError::transport(context.clone(), REFUSED);
305        assert!(is_dead_pocket_ic_transport_error(&error));
306        let error = super::super::CandidCallError::decode(context, 0, "channel closed");
307        assert!(!is_dead_pocket_ic_transport_error(&error));
308    }
309}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
