ic_testkit/pic/
transport.rs

1use std::any::Any;
2
3/// Captured PocketIC operation failure, retaining transport classification.
4#[derive(Clone, Debug, Eq, PartialEq)]
5pub struct PocketIcOperationError {
6    message: String,
7    transport: bool,
8}
9
10impl PocketIcOperationError {
11    /// Capture an upstream PocketIC panic message and classify its transport.
12    #[must_use]
13    pub fn new(message: impl Into<String>) -> Self {
14        let message = message.into();
15        let transport = is_dead_instance_transport_error(&message);
16        Self { message, transport }
17    }
18
19    pub(super) fn from_panic(payload: &(dyn Any + Send)) -> Self {
20        Self::new(panic_payload_to_string(payload))
21    }
22
23    /// Read the original upstream message without contextual wrappers.
24    #[must_use]
25    pub fn message(&self) -> &str {
26        &self.message
27    }
28
29    /// Whether the message matches a recognized PocketIC transport failure.
30    #[must_use]
31    pub const fn is_transport(&self) -> bool {
32        self.transport
33    }
34}
35
36impl std::fmt::Display for PocketIcOperationError {
37    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
38        formatter.write_str(&self.message)
39    }
40}
41
42impl std::error::Error for PocketIcOperationError {}
43
44// Extract a stable string message from one panic payload.
45pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
46    if let Some(message) = payload.downcast_ref::<String>() {
47        return message.clone();
48    }
49    if let Some(message) = payload.downcast_ref::<&'static str>() {
50        return (*message).to_string();
51    }
52
53    "non-string panic payload".to_string()
54}
55
56// Check whether one panic payload belongs to the dead-instance transport class
57// without consuming it, so callers can still resume the original panic.
58pub(super) fn panic_is_dead_instance_transport(payload: &(dyn Any + Send)) -> bool {
59    payload
60        .downcast_ref::<String>()
61        .map(String::as_str)
62        .or_else(|| payload.downcast_ref::<&'static str>().copied())
63        .is_some_and(is_dead_instance_transport_error)
64}
65
66/// Recognize transport failures in PocketIC's unstructured request errors.
67///
68/// The complete error source chain is inspected so a recipe can classify its
69/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
70/// requires a reqwest debug error with a PocketIC instance URL and a recognized
71/// transport source, optionally prefixed by PocketIC's HTTP panic context.
72/// Generic application messages and bare I/O error kinds do not qualify.
73/// Testkit [`super::CandidCallError`] and [`PocketIcOperationError`] causes
74/// retain their transport classification through contextual display wrappers.
75/// Snapshot and installation failures expose their operation cause through
76/// the error source chain.
77///
78/// Use this only for errors originating in a PocketIC operation. This is a
79/// message-based heuristic, not proof that the server or instance has died;
80/// another service or application can reproduce the same text. PocketIC does
81/// not yet expose a structured transport error.
82#[must_use]
83pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
84    let mut current = Some(error);
85    while let Some(candidate) = current {
86        if candidate
87            .downcast_ref::<PocketIcOperationError>()
88            .is_some_and(PocketIcOperationError::is_transport)
89        {
90            return true;
91        }
92        if candidate
93            .downcast_ref::<super::CandidCallError>()
94            .is_some_and(|error| error.kind() == super::CandidCallErrorKind::Transport)
95        {
96            return true;
97        }
98        if is_dead_instance_transport_error(&candidate.to_string()) {
99            return true;
100        }
101        current = candidate.source();
102    }
103    false
104}
105
106// Recognize maintained PocketIC request-error shapes for restore recovery.
107pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
108    let message = message
109        .strip_prefix("HTTP failure: ")
110        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
111        .unwrap_or(message);
112    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
113        return false;
114    };
115    let Some((kind, request)) = error.split_once(", url: \"") else {
116        return false;
117    };
118    if !matches!(kind, "Request" | "Body" | "Decode") {
119        return false;
120    }
121    let Some((url, source)) = request.split_once("\", source: ") else {
122        return false;
123    };
124    let Some(source) = source.strip_suffix(" }") else {
125        return false;
126    };
127    let Some(authority_and_path) = url
128        .strip_prefix("http://")
129        .or_else(|| url.strip_prefix("https://"))
130    else {
131        return false;
132    };
133    let Some((authority, path)) = authority_and_path.split_once('/') else {
134        return false;
135    };
136    let Some(instance_path) = path.strip_prefix("instances/") else {
137        return false;
138    };
139    let instance_id = instance_path.split('/').next().unwrap_or_default();
140    if authority.is_empty()
141        || instance_id.is_empty()
142        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
143    {
144        return false;
145    }
146    let source = source
147        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
148        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
149        .and_then(|source| source.strip_suffix(')'))
150        .unwrap_or(source);
151    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
152        || matches!(
153            source,
154            "hyper::Error(IncompleteMessage)"
155                | "hyper::Error(ChannelClosed)"
156                | "hyper::Error(Canceled, \"connection closed before message completed\")"
157        )
158}
159
160#[cfg(test)]
161mod tests {
162    use super::{
163        PocketIcOperationError, is_dead_instance_transport_error,
164        is_dead_pocket_ic_transport_error, panic_is_dead_instance_transport,
165    };
166
167    #[derive(Debug)]
168    struct WrapperError(std::io::Error);
169
170    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
171    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
172
173    impl std::fmt::Display for WrapperError {
174        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
175            formatter.write_str("wrapped PocketIC request failed")
176        }
177    }
178
179    impl std::error::Error for WrapperError {
180        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
181            Some(&self.0)
182        }
183    }
184
185    #[test]
186    fn dead_instance_transport_error_detects_connection_refused() {
187        assert!(is_dead_instance_transport_error(REFUSED));
188        assert!(is_dead_instance_transport_error(&format!(
189            "HTTP failure: {REFUSED}"
190        )));
191    }
192
193    #[test]
194    fn dead_instance_transport_error_detects_incomplete_message() {
195        assert!(is_dead_instance_transport_error(INCOMPLETE));
196        assert!(is_dead_instance_transport_error(&format!(
197            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
198        )));
199    }
200
201    #[test]
202    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
203        let classified = PocketIcOperationError::new(INCOMPLETE);
204
205        assert!(classified.is_transport());
206        assert!(panic_is_dead_instance_transport(&INCOMPLETE));
207    }
208
209    #[test]
210    fn public_classifier_inspects_the_error_source_chain() {
211        let dead = WrapperError(std::io::Error::other(REFUSED));
212        assert!(is_dead_pocket_ic_transport_error(&dead));
213
214        let unrelated = WrapperError(std::io::Error::other("request rejected"));
215        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
216    }
217
218    #[test]
219    fn snapshot_and_install_wrappers_preserve_transport_causes() {
220        use super::super::{CanisterInstallError, CanisterInstallPhase, ControllerSnapshotError};
221        let canister_id = candid::Principal::anonymous();
222        for message in [REFUSED, "unrelated application panic"] {
223            let expected = message == REFUSED;
224            let capture = ControllerSnapshotError::CapturePanicked {
225                canister_id,
226                source: PocketIcOperationError::new(message),
227                cleanup_failures: vec![],
228            };
229            let restore = ControllerSnapshotError::RestorePanicked {
230                canister_id,
231                source: PocketIcOperationError::new(message),
232            };
233            for snapshot in [capture, restore] {
234                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
235                let wrapper = WrapperError(std::io::Error::other(snapshot));
236                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
237            }
238            for phase in [
239                CanisterInstallPhase::CreateCanister,
240                CanisterInstallPhase::AddCycles,
241                CanisterInstallPhase::InstallCode,
242            ] {
243                let install = CanisterInstallError::new(
244                    phase,
245                    None,
246                    Some("fixture".into()),
247                    PocketIcOperationError::new(message),
248                );
249                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
250                let wrapper = WrapperError(std::io::Error::other(install));
251                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
252            }
253        }
254    }
255
256    #[test]
257    fn unrelated_and_quoted_transport_text_does_not_qualify() {
258        for message in [
259            "application worker channel closed",
260            "fixture expected ConnectionRefused but observed another result",
261            "tcp connect error",
262            "connection closed before message completed",
263            "IncompleteMessage",
264            &format!("fixture quoted {REFUSED}"),
265            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
266            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
267            &INCOMPLETE.replace(
268                "hyper::Error(IncompleteMessage)",
269                "Custom(\"channel closed\")",
270            ),
271            &INCOMPLETE.replace(
272                "hyper::Error(IncompleteMessage)",
273                "Custom(\"hyper::Error(IncompleteMessage)\")",
274            ),
275        ] {
276            let error = WrapperError(std::io::Error::other(message.to_owned()));
277            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
278            assert!(!PocketIcOperationError::new(message).is_transport());
279            assert!(!panic_is_dead_instance_transport(&message.to_owned()));
280        }
281        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
282        assert!(!is_dead_pocket_ic_transport_error(&refused));
283        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
284    }
285
286    #[test]
287    fn structured_channel_closure_on_an_instance_request_qualifies() {
288        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
289        assert!(is_dead_instance_transport_error(&message));
290    }
291
292    #[test]
293    fn contextual_call_errors_use_their_structured_kind() {
294        let context = super::super::CandidCallContext::new(
295            "query_call",
296            candid::Principal::anonymous(),
297            candid::Principal::anonymous(),
298            "get",
299        );
300        let error = super::super::CandidCallError::transport(context.clone(), REFUSED);
301        assert!(is_dead_pocket_ic_transport_error(&error));
302        let error = super::super::CandidCallError::decode(context, 0, "channel closed");
303        assert!(!is_dead_pocket_ic_transport_error(&error));
304    }
305}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
