ic_testkit/pic/
transport.rs

1use std::{
2    any::Any,
3    panic::{AssertUnwindSafe, catch_unwind, resume_unwind},
4};
5
6use ic_platform::pocket_ic::RejectResponse;
7
8/// Captured PocketIC operation failure, retaining transport classification.
9#[derive(Clone, Debug, Eq, PartialEq)]
10pub struct PocketIcOperationError {
11    message: String,
12    transport: bool,
13}
14
15impl PocketIcOperationError {
16    /// Capture an upstream PocketIC panic message and classify its transport.
17    #[must_use]
18    pub fn new(message: impl Into<String>) -> Self {
19        let message = message.into();
20        let transport = is_dead_instance_transport_error(&message);
21        Self { message, transport }
22    }
23
24    /// Read the original upstream message without contextual wrappers.
25    #[must_use]
26    pub fn message(&self) -> &str {
27        &self.message
28    }
29
30    /// Whether the message matches a recognized PocketIC transport failure.
31    #[must_use]
32    pub const fn is_transport(&self) -> bool {
33        self.transport
34    }
35}
36
37// Only callers that intentionally capture every unwind use this boundary.
38// Typed calls retain their separate policy of resuming unrelated panics.
39pub(super) fn capture_operation<T>(
40    operation: impl FnOnce() -> T,
41) -> Result<T, PocketIcOperationError> {
42    catch_unwind(AssertUnwindSafe(operation))
43        .map_err(|payload| PocketIcOperationError::new(panic_payload_to_string(payload.as_ref())))
44}
45
46impl std::fmt::Display for PocketIcOperationError {
47    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48        formatter.write_str(&self.message)
49    }
50}
51
52impl std::error::Error for PocketIcOperationError {}
53
54/// Execute one synchronous PocketIC operation, returning recognized transport panics.
55///
56/// Successful values are returned unchanged, including any upstream `Result` and
57/// its original rejection. Unrelated panics resume with their original payload.
58/// The closure is invoked once: this helper never retries or replays an operation.
59/// A transport failure does not establish whether an update was accepted or
60/// whether an instance died; callers retain recovery and reconciliation policy.
61///
62/// Recognition uses the maintained panic shapes of the selected PocketIC client.
63/// It cannot recover the typed cause discarded upstream, suppress the panic hook,
64/// bound a blocked request, or catch errors while polling a returned future.
65///
66/// ```no_run
67/// use ic_platform::pocket_ic::PocketIc;
68/// use ic_testkit::pic::{PocketIcOperationError, try_pocket_ic_operation};
69///
70/// fn tick(pic: &PocketIc) -> Result<(), PocketIcOperationError> {
71///     try_pocket_ic_operation(|| pic.tick())
72/// }
73/// ```
74///
75/// # Panics
76/// Unrecognized panic payloads continue unwinding.
77pub fn try_pocket_ic_operation<T>(
78    operation: impl FnOnce() -> T,
79) -> Result<T, PocketIcOperationError> {
80    match catch_unwind(AssertUnwindSafe(operation)) {
81        Ok(value) => Ok(value),
82        Err(payload) => {
83            let source = PocketIcOperationError::new(panic_payload_to_string(payload.as_ref()));
84            if source.is_transport() {
85                Err(source)
86            } else {
87                resume_unwind(payload)
88            }
89        }
90    }
91}
92
93/// A rejected management call or a captured PocketIC operation panic.
94#[non_exhaustive]
95#[derive(Debug)]
96pub enum PocketIcCallFailure {
97    /// PocketIC returned a structured management-canister rejection.
98    Rejected(RejectResponse),
99    /// Captured PocketIC panic, retaining its original transport classification.
100    Operation(PocketIcOperationError),
101}
102
103impl std::fmt::Display for PocketIcCallFailure {
104    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
105        match self {
106            Self::Rejected(response) => write!(formatter, "rejected: {response:?}"),
107            Self::Operation(source) if source.is_transport() => {
108                write!(formatter, "PocketIC instance unavailable: {source}")
109            }
110            Self::Operation(source) => write!(formatter, "panicked: {source}"),
111        }
112    }
113}
114
115impl std::error::Error for PocketIcCallFailure {
116    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
117        match self {
118            Self::Rejected(_) => None,
119            Self::Operation(source) => Some(source),
120        }
121    }
122}
123
124pub(super) fn capture_call<T>(
125    call: impl FnOnce() -> Result<T, RejectResponse>,
126) -> Result<T, PocketIcCallFailure> {
127    match capture_operation(call) {
128        Ok(Ok(value)) => Ok(value),
129        Ok(Err(response)) => Err(PocketIcCallFailure::Rejected(response)),
130        Err(source) => Err(PocketIcCallFailure::Operation(source)),
131    }
132}
133
134// Extract a stable string message from one panic payload.
135pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
136    if let Some(message) = payload.downcast_ref::<String>() {
137        return message.clone();
138    }
139    if let Some(message) = payload.downcast_ref::<&'static str>() {
140        return (*message).to_string();
141    }
142
143    "non-string panic payload".to_string()
144}
145
146/// Recognize PocketIC transport failures in typed causes and request-error text.
147///
148/// The complete error source chain is inspected so a recipe can classify its
149/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
150/// preserves the classification already captured by testkit's structured errors.
151/// Testkit's optional Wire call errors, [`super::PocketIcCallFailure`], and
152/// [`PocketIcOperationError`] causes retain their transport classification through
153/// contextual display wrappers, including nested [`std::io::Error`] values.
154/// Snapshot and installation failures expose their operation cause through
155/// the error source chain.
156///
157/// For other errors, message recognition requires a reqwest debug error with a
158/// PocketIC instance URL and a recognized transport source, optionally prefixed
159/// by PocketIC's HTTP panic context. Generic application messages and bare I/O
160/// error kinds do not qualify.
161/// Recognized sources include OS connection resets during an HTTP request.
162///
163/// Use this only for errors originating in a PocketIC operation. This is a
164/// message-based heuristic, not proof that the server or instance has died;
165/// another service or application can reproduce the same text. PocketIC does
166/// not yet expose a structured transport error.
167#[must_use]
168pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
169    let mut current = Some(error);
170    while let Some(candidate) = current {
171        // Owned codec errors and canister rejections are not transport failures.
172        #[cfg(feature = "wire")]
173        if let Some(error) = candidate.downcast_ref::<super::WireCallError>() {
174            return match error {
175                super::WireCallError::Encode { .. } | super::WireCallError::Decode { .. } => false,
176                super::WireCallError::Call { source, .. } => matches!(
177                    source.as_ref(),
178                    PocketIcCallFailure::Operation(error) if error.is_transport()
179                ),
180            };
181        }
182        if let Some(failure) = candidate.downcast_ref::<PocketIcCallFailure>() {
183            return matches!(failure, PocketIcCallFailure::Operation(error) if error.is_transport());
184        }
185        if candidate
186            .downcast_ref::<PocketIcOperationError>()
187            .is_some_and(PocketIcOperationError::is_transport)
188        {
189            return true;
190        }
191        if is_dead_instance_transport_error(&candidate.to_string()) {
192            return true;
193        }
194        // io::Error::source delegates to the contained error's source, skipping
195        // the contained error itself. Inspect it before following its causes so
196        // contextual errors keep their own structured classification.
197        current = if let Some(inner) = candidate
198            .downcast_ref::<std::io::Error>()
199            .and_then(std::io::Error::get_ref)
200        {
201            Some(inner)
202        } else {
203            candidate.source()
204        };
205    }
206    false
207}
208
209// Recognize maintained PocketIC request-error shapes for restore recovery.
210pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
211    let message = message
212        .strip_prefix("HTTP failure: ")
213        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
214        .unwrap_or(message);
215    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
216        return false;
217    };
218    let Some((kind, request)) = error.split_once(", url: \"") else {
219        return false;
220    };
221    if !matches!(kind, "Request" | "Body" | "Decode") {
222        return false;
223    }
224    let Some((url, source)) = request.split_once("\", source: ") else {
225        return false;
226    };
227    let Some(source) = source.strip_suffix(" }") else {
228        return false;
229    };
230    let Some(authority_and_path) = url
231        .strip_prefix("http://")
232        .or_else(|| url.strip_prefix("https://"))
233    else {
234        return false;
235    };
236    let Some((authority, path)) = authority_and_path.split_once('/') else {
237        return false;
238    };
239    let Some(instance_path) = path.strip_prefix("instances/") else {
240        return false;
241    };
242    let instance_id = instance_path.split('/').next().unwrap_or_default();
243    if authority.is_empty()
244        || instance_id.is_empty()
245        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
246    {
247        return false;
248    }
249    let source = source
250        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
251        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
252        .and_then(|source| source.strip_suffix(')'))
253        .unwrap_or(source);
254    let reset = source
255        .strip_prefix("hyper::Error(Io, Os { code: ")
256        .and_then(|source| source.split_once(", kind: ConnectionReset, message: \""))
257        .is_some_and(|(code, message)| code.parse::<i32>().is_ok() && message.ends_with("\" })"));
258    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
259        || reset
260        || matches!(
261            source,
262            "hyper::Error(IncompleteMessage)"
263                | "hyper::Error(ChannelClosed)"
264                | "hyper::Error(Canceled, \"connection closed before message completed\")"
265        )
266}
267
268#[cfg(test)]
269mod tests {
270    use super::super::{CanisterInstallError, ControllerSnapshotError};
271    use super::{
272        PocketIcCallFailure, PocketIcOperationError, capture_operation,
273        is_dead_instance_transport_error, is_dead_pocket_ic_transport_error,
274        try_pocket_ic_operation,
275    };
276
277    #[derive(Debug)]
278    struct WrapperError(std::io::Error);
279
280    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
281    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
282    const RESET: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:49238/instances/0/update/submit_ingress_message\", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(Io, Os { code: 54, kind: ConnectionReset, message: \"Connection reset by peer\" })) }";
283
284    #[test]
285    fn raw_operations_preserve_values_and_rejections() {
286        assert_eq!(try_pocket_ic_operation(|| 42), Ok(42));
287        let rejection = ic_platform::pocket_ic::RejectResponse {
288            reject_code: ic_platform::pocket_ic::RejectCode::CanisterReject,
289            reject_message: "application rejection".into(),
290            error_code: ic_platform::pocket_ic::ErrorCode::CanisterRejectedMessage,
291            certified: false,
292        };
293        let result = try_pocket_ic_operation(|| Err::<(), _>(rejection))
294            .expect("returned rejection is not a transport panic");
295        let rejection = result.expect_err("retain the original rejection");
296        assert_eq!(rejection.reject_message, "application rejection");
297        assert_eq!(
298            rejection.reject_code,
299            ic_platform::pocket_ic::RejectCode::CanisterReject
300        );
301    }
302
303    #[test]
304    fn raw_transport_failure_is_returned_after_one_attempt() {
305        let mut attempts = 0;
306        let error = try_pocket_ic_operation::<()>(|| {
307            attempts += 1;
308            panic!("{REFUSED}");
309        })
310        .expect_err("transport panic must return an error");
311        assert_eq!(attempts, 1);
312        assert_eq!(error.message(), REFUSED);
313        assert!(error.is_transport());
314    }
315
316    impl std::fmt::Display for WrapperError {
317        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
318            formatter.write_str("wrapped PocketIC request failed")
319        }
320    }
321
322    impl std::error::Error for WrapperError {
323        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
324            Some(&self.0)
325        }
326    }
327
328    #[test]
329    fn dead_instance_transport_error_detects_connection_refused() {
330        assert!(is_dead_instance_transport_error(REFUSED));
331        assert!(is_dead_instance_transport_error(&format!(
332            "HTTP failure: {REFUSED}"
333        )));
334    }
335
336    #[test]
337    fn dead_instance_transport_error_detects_incomplete_message() {
338        assert!(is_dead_instance_transport_error(INCOMPLETE));
339        assert!(is_dead_instance_transport_error(&format!(
340            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
341        )));
342    }
343
344    #[test]
345    fn dead_instance_transport_error_detects_native_connection_reset() {
346        for message in [RESET.to_owned(), RESET.replace("code: 54", "code: 104")] {
347            assert!(is_dead_instance_transport_error(&message));
348            assert!(PocketIcOperationError::new(format!("HTTP failure: {message}")).is_transport());
349            assert!(is_dead_instance_transport_error(&format!(
350                "called `Result::unwrap()` on an `Err` value: {message}"
351            )));
352        }
353    }
354
355    #[test]
356    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
357        let classified = capture_operation::<()>(|| panic!("{INCOMPLETE}"))
358            .expect_err("operation panic must be captured");
359
360        assert_eq!(classified.message(), INCOMPLETE);
361        assert!(classified.is_transport());
362        let non_string = capture_operation::<()>(|| std::panic::panic_any(7_u8))
363            .expect_err("non-string panic must be captured");
364        assert_eq!(non_string.message(), "non-string panic payload");
365        assert!(!non_string.is_transport());
366    }
367
368    #[test]
369    fn public_classifier_inspects_the_error_source_chain() {
370        let dead = WrapperError(std::io::Error::other(REFUSED));
371        assert!(is_dead_pocket_ic_transport_error(&dead));
372
373        let unrelated = WrapperError(std::io::Error::other("request rejected"));
374        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
375    }
376
377    #[test]
378    fn snapshot_and_install_wrappers_preserve_transport_causes() {
379        let canister_id = ic_platform::principal::ExternalPrincipal::anonymous();
380        for message in [REFUSED, RESET, "unrelated application panic"] {
381            let expected = message != "unrelated application panic";
382            let capture = ControllerSnapshotError::CaptureFailed {
383                canister_id,
384                sender: None,
385                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
386                    message,
387                ))),
388                cleanup_failures: vec![],
389            };
390            let restore = ControllerSnapshotError::RestoreFailed {
391                canister_id,
392                sender: None,
393                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
394                    message,
395                ))),
396            };
397            for snapshot in [capture, restore] {
398                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
399                let wrapper = WrapperError(std::io::Error::other(snapshot));
400                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
401            }
402            for install in [
403                CanisterInstallError::CreateCanister {
404                    label: Some("fixture".into()),
405                    source: PocketIcOperationError::new(message),
406                },
407                CanisterInstallError::AddCycles {
408                    canister_id,
409                    label: Some("fixture".into()),
410                    source: PocketIcOperationError::new(message),
411                },
412                CanisterInstallError::InstallCode {
413                    canister_id,
414                    label: Some("fixture".into()),
415                    source: PocketIcOperationError::new(message),
416                },
417            ] {
418                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
419                let wrapper = WrapperError(std::io::Error::other(install));
420                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
421            }
422        }
423    }
424
425    #[test]
426    fn unrelated_and_quoted_transport_text_does_not_qualify() {
427        for message in [
428            "application worker channel closed",
429            "fixture expected ConnectionRefused but observed another result",
430            "tcp connect error",
431            "connection closed before message completed",
432            "IncompleteMessage",
433            &format!("fixture quoted {REFUSED}"),
434            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
435            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
436            &INCOMPLETE.replace(
437                "hyper::Error(IncompleteMessage)",
438                "Custom(\"channel closed\")",
439            ),
440            &INCOMPLETE.replace(
441                "hyper::Error(IncompleteMessage)",
442                "Custom(\"hyper::Error(IncompleteMessage)\")",
443            ),
444            &format!("fixture quoted {RESET}"),
445            &RESET.replace("/instances/0/", "/application/worker/"),
446            &RESET.replace("/instances/0/", "/instances/quoted/"),
447            &RESET.replace("kind: ConnectionReset,", "kind: ConnectionAborted,"),
448            &RESET.replace("code: 54", "code: quoted"),
449            &RESET.replace("hyper::Error(Io, Os {", "Custom(Os {"),
450        ] {
451            let error = WrapperError(std::io::Error::other(message.to_owned()));
452            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
453            assert!(!PocketIcOperationError::new(message).is_transport());
454        }
455        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
456        assert!(!is_dead_pocket_ic_transport_error(&refused));
457        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
458        let reset = std::io::Error::from(std::io::ErrorKind::ConnectionReset);
459        assert!(!is_dead_pocket_ic_transport_error(&reset));
460        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(reset)));
461    }
462
463    #[test]
464    fn structured_channel_closure_on_an_instance_request_qualifies() {
465        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
466        assert!(is_dead_instance_transport_error(&message));
467    }
468
469    #[cfg(feature = "wire")]
470    #[test]
471    fn contextual_call_errors_use_their_structured_kind() {
472        use super::super::{WireCallContext, WireCallError};
473        let context = WireCallContext {
474            operation: "query_call",
475            canister_id: ic_platform::principal::ExternalPrincipal::anonymous(),
476            caller: ic_platform::principal::ExternalPrincipal::anonymous(),
477            method: REFUSED,
478            method_id: 1,
479        };
480        for (error, expected) in [
481            (
482                WireCallError::Call {
483                    context: Box::new(context.clone()),
484                    source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
485                        REFUSED,
486                    ))),
487                },
488                true,
489            ),
490            (
491                WireCallError::Decode {
492                    context: Box::new(context.clone()),
493                    bytes: 0,
494                    source: ic_wire::Error::Malformed,
495                },
496                false,
497            ),
498            (
499                WireCallError::Encode {
500                    context: Box::new(context.clone()),
501                    source: ic_wire::Error::Limit,
502                },
503                false,
504            ),
505            (
506                WireCallError::Call {
507                    context: Box::new(context),
508                    source: Box::new(PocketIcCallFailure::Rejected(
509                        ic_platform::pocket_ic::RejectResponse {
510                            reject_code: ic_platform::pocket_ic::RejectCode::CanisterReject,
511                            reject_message: REFUSED.into(),
512                            error_code: ic_platform::pocket_ic::ErrorCode::CanisterRejectedMessage,
513                            certified: false,
514                        },
515                    )),
516                },
517                false,
518            ),
519        ] {
520            assert_eq!(is_dead_pocket_ic_transport_error(&error), expected);
521            let wrapped = WrapperError(std::io::Error::other(std::io::Error::other(error)));
522            assert_eq!(is_dead_pocket_ic_transport_error(&wrapped), expected);
523        }
524    }
525}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
