ic_testkit/pic/
transport.rs

1use std::{
2    any::Any,
3    panic::{AssertUnwindSafe, catch_unwind, resume_unwind},
4};
5
6use ic_platform::pocket_ic::RejectResponse;
7
8/// Captured PocketIC operation failure, retaining transport classification.
9#[derive(Clone, Debug, Eq, PartialEq)]
10pub struct PocketIcOperationError {
11    message: String,
12    transport: bool,
13}
14
15impl PocketIcOperationError {
16    /// Capture an upstream PocketIC panic message and classify its transport.
17    #[must_use]
18    pub fn new(message: impl Into<String>) -> Self {
19        let message = message.into();
20        let transport = is_dead_instance_transport_error(&message);
21        Self { message, transport }
22    }
23
24    /// Read the original upstream message without contextual wrappers.
25    #[must_use]
26    pub fn message(&self) -> &str {
27        &self.message
28    }
29
30    /// Whether the message matches a recognized PocketIC transport failure.
31    #[must_use]
32    pub const fn is_transport(&self) -> bool {
33        self.transport
34    }
35}
36
37// Only callers that intentionally capture every unwind use this boundary.
38// Typed calls retain their separate policy of resuming unrelated panics.
39pub(super) fn capture_operation<T>(
40    operation: impl FnOnce() -> T,
41) -> Result<T, PocketIcOperationError> {
42    catch_unwind(AssertUnwindSafe(operation))
43        .map_err(|payload| PocketIcOperationError::new(panic_payload_to_string(payload.as_ref())))
44}
45
46impl std::fmt::Display for PocketIcOperationError {
47    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48        formatter.write_str(&self.message)
49    }
50}
51
52impl std::error::Error for PocketIcOperationError {}
53
54/// Execute one synchronous PocketIC operation, returning recognized transport panics.
55///
56/// Successful values are returned unchanged, including any upstream `Result` and
57/// its original rejection. Unrelated panics resume with their original payload.
58/// The closure is invoked once: this helper never retries or replays an operation.
59/// A transport failure does not establish whether an update was accepted or
60/// whether an instance died; callers retain recovery and reconciliation policy.
61///
62/// Recognition uses the maintained panic shapes of the selected PocketIC client.
63/// It cannot recover the typed cause discarded upstream, suppress the panic hook,
64/// bound a blocked request, or catch errors while polling a returned future.
65///
66/// ```no_run
67/// use ic_testkit::pic::{PocketIc, PocketIcOperationError, try_pocket_ic_operation};
68///
69/// fn tick(pic: &PocketIc) -> Result<(), PocketIcOperationError> {
70///     try_pocket_ic_operation(|| pic.tick())
71/// }
72/// ```
73///
74/// # Panics
75/// Unrecognized panic payloads continue unwinding.
76pub fn try_pocket_ic_operation<T>(
77    operation: impl FnOnce() -> T,
78) -> Result<T, PocketIcOperationError> {
79    match catch_unwind(AssertUnwindSafe(operation)) {
80        Ok(value) => Ok(value),
81        Err(payload) => {
82            let source = PocketIcOperationError::new(panic_payload_to_string(payload.as_ref()));
83            if source.is_transport() {
84                Err(source)
85            } else {
86                resume_unwind(payload)
87            }
88        }
89    }
90}
91
92/// A rejected management call or a captured PocketIC operation panic.
93#[non_exhaustive]
94#[derive(Debug)]
95pub enum PocketIcCallFailure {
96    /// PocketIC returned a structured management-canister rejection.
97    Rejected(RejectResponse),
98    /// Captured PocketIC panic, retaining its original transport classification.
99    Operation(PocketIcOperationError),
100}
101
102impl std::fmt::Display for PocketIcCallFailure {
103    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104        match self {
105            Self::Rejected(response) => write!(formatter, "rejected: {response:?}"),
106            Self::Operation(source) if source.is_transport() => {
107                write!(formatter, "PocketIC instance unavailable: {source}")
108            }
109            Self::Operation(source) => write!(formatter, "panicked: {source}"),
110        }
111    }
112}
113
114impl std::error::Error for PocketIcCallFailure {
115    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
116        match self {
117            Self::Rejected(_) => None,
118            Self::Operation(source) => Some(source),
119        }
120    }
121}
122
123pub(super) fn capture_call<T>(
124    call: impl FnOnce() -> Result<T, RejectResponse>,
125) -> Result<T, PocketIcCallFailure> {
126    match capture_operation(call) {
127        Ok(Ok(value)) => Ok(value),
128        Ok(Err(response)) => Err(PocketIcCallFailure::Rejected(response)),
129        Err(source) => Err(PocketIcCallFailure::Operation(source)),
130    }
131}
132
133// Extract a stable string message from one panic payload.
134pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
135    if let Some(message) = payload.downcast_ref::<String>() {
136        return message.clone();
137    }
138    if let Some(message) = payload.downcast_ref::<&'static str>() {
139        return (*message).to_string();
140    }
141
142    "non-string panic payload".to_string()
143}
144
145/// Recognize PocketIC transport failures in typed causes and request-error text.
146///
147/// The complete error source chain is inspected so a recipe can classify its
148/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
149/// preserves the classification already captured by testkit's structured errors.
150/// Testkit's optional Wire call errors, [`super::PocketIcCallFailure`], and
151/// [`PocketIcOperationError`] causes retain their transport classification through
152/// contextual display wrappers, including nested [`std::io::Error`] values.
153/// Snapshot and installation failures expose their operation cause through
154/// the error source chain.
155///
156/// For other errors, message recognition requires a reqwest debug error with a
157/// PocketIC instance URL and a recognized transport source, optionally prefixed
158/// by PocketIC's HTTP panic context. Generic application messages and bare I/O
159/// error kinds do not qualify.
160/// Recognized sources include OS connection resets during an HTTP request.
161///
162/// Use this only for errors originating in a PocketIC operation. This is a
163/// message-based heuristic, not proof that the server or instance has died;
164/// another service or application can reproduce the same text. PocketIC does
165/// not yet expose a structured transport error.
166#[must_use]
167pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
168    let mut current = Some(error);
169    while let Some(candidate) = current {
170        // Owned codec errors and canister rejections are not transport failures.
171        #[cfg(feature = "wire")]
172        if let Some(error) = candidate.downcast_ref::<super::WireCallError>() {
173            return match error {
174                super::WireCallError::Encode { .. } | super::WireCallError::Decode { .. } => false,
175                super::WireCallError::Call { source, .. } => matches!(
176                    source.as_ref(),
177                    PocketIcCallFailure::Operation(error) if error.is_transport()
178                ),
179            };
180        }
181        if let Some(failure) = candidate.downcast_ref::<PocketIcCallFailure>() {
182            return matches!(failure, PocketIcCallFailure::Operation(error) if error.is_transport());
183        }
184        if candidate
185            .downcast_ref::<PocketIcOperationError>()
186            .is_some_and(PocketIcOperationError::is_transport)
187        {
188            return true;
189        }
190        if is_dead_instance_transport_error(&candidate.to_string()) {
191            return true;
192        }
193        // io::Error::source delegates to the contained error's source, skipping
194        // the contained error itself. Inspect it before following its causes so
195        // contextual errors keep their own structured classification.
196        current = if let Some(inner) = candidate
197            .downcast_ref::<std::io::Error>()
198            .and_then(std::io::Error::get_ref)
199        {
200            Some(inner)
201        } else {
202            candidate.source()
203        };
204    }
205    false
206}
207
208// Recognize maintained PocketIC request-error shapes for restore recovery.
209pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
210    let message = message
211        .strip_prefix("HTTP failure: ")
212        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
213        .unwrap_or(message);
214    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
215        return false;
216    };
217    let Some((kind, request)) = error.split_once(", url: \"") else {
218        return false;
219    };
220    if !matches!(kind, "Request" | "Body" | "Decode") {
221        return false;
222    }
223    let Some((url, source)) = request.split_once("\", source: ") else {
224        return false;
225    };
226    let Some(source) = source.strip_suffix(" }") else {
227        return false;
228    };
229    let Some(authority_and_path) = url
230        .strip_prefix("http://")
231        .or_else(|| url.strip_prefix("https://"))
232    else {
233        return false;
234    };
235    let Some((authority, path)) = authority_and_path.split_once('/') else {
236        return false;
237    };
238    let Some(instance_path) = path.strip_prefix("instances/") else {
239        return false;
240    };
241    let instance_id = instance_path.split('/').next().unwrap_or_default();
242    if authority.is_empty()
243        || instance_id.is_empty()
244        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
245    {
246        return false;
247    }
248    let source = source
249        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
250        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
251        .and_then(|source| source.strip_suffix(')'))
252        .unwrap_or(source);
253    let reset = source
254        .strip_prefix("hyper::Error(Io, Os { code: ")
255        .and_then(|source| source.split_once(", kind: ConnectionReset, message: \""))
256        .is_some_and(|(code, message)| code.parse::<i32>().is_ok() && message.ends_with("\" })"));
257    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
258        || reset
259        || matches!(
260            source,
261            "hyper::Error(IncompleteMessage)"
262                | "hyper::Error(ChannelClosed)"
263                | "hyper::Error(Canceled, \"connection closed before message completed\")"
264        )
265}
266
267#[cfg(test)]
268mod tests {
269    use super::super::{CanisterInstallError, ControllerSnapshotError};
270    use super::{
271        PocketIcCallFailure, PocketIcOperationError, capture_operation,
272        is_dead_instance_transport_error, is_dead_pocket_ic_transport_error,
273        try_pocket_ic_operation,
274    };
275
276    #[derive(Debug)]
277    struct WrapperError(std::io::Error);
278
279    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
280    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
281    const RESET: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:49238/instances/0/update/submit_ingress_message\", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(Io, Os { code: 54, kind: ConnectionReset, message: \"Connection reset by peer\" })) }";
282
283    #[test]
284    fn raw_operations_preserve_values_and_rejections() {
285        assert_eq!(try_pocket_ic_operation(|| 42), Ok(42));
286        let rejection = ic_platform::pocket_ic::RejectResponse {
287            reject_code: ic_platform::pocket_ic::RejectCode::CanisterReject,
288            reject_message: "application rejection".into(),
289            error_code: ic_platform::pocket_ic::ErrorCode::CanisterRejectedMessage,
290            certified: false,
291        };
292        let result = try_pocket_ic_operation(|| Err::<(), _>(rejection))
293            .expect("returned rejection is not a transport panic");
294        let rejection = result.expect_err("retain the original rejection");
295        assert_eq!(rejection.reject_message, "application rejection");
296        assert_eq!(
297            rejection.reject_code,
298            ic_platform::pocket_ic::RejectCode::CanisterReject
299        );
300    }
301
302    #[test]
303    fn raw_transport_failure_is_returned_after_one_attempt() {
304        let mut attempts = 0;
305        let error = try_pocket_ic_operation::<()>(|| {
306            attempts += 1;
307            panic!("{REFUSED}");
308        })
309        .expect_err("transport panic must return an error");
310        assert_eq!(attempts, 1);
311        assert_eq!(error.message(), REFUSED);
312        assert!(error.is_transport());
313    }
314
315    impl std::fmt::Display for WrapperError {
316        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
317            formatter.write_str("wrapped PocketIC request failed")
318        }
319    }
320
321    impl std::error::Error for WrapperError {
322        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
323            Some(&self.0)
324        }
325    }
326
327    #[test]
328    fn dead_instance_transport_error_detects_connection_refused() {
329        assert!(is_dead_instance_transport_error(REFUSED));
330        assert!(is_dead_instance_transport_error(&format!(
331            "HTTP failure: {REFUSED}"
332        )));
333    }
334
335    #[test]
336    fn dead_instance_transport_error_detects_incomplete_message() {
337        assert!(is_dead_instance_transport_error(INCOMPLETE));
338        assert!(is_dead_instance_transport_error(&format!(
339            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
340        )));
341    }
342
343    #[test]
344    fn dead_instance_transport_error_detects_native_connection_reset() {
345        for message in [RESET.to_owned(), RESET.replace("code: 54", "code: 104")] {
346            assert!(is_dead_instance_transport_error(&message));
347            assert!(PocketIcOperationError::new(format!("HTTP failure: {message}")).is_transport());
348            assert!(is_dead_instance_transport_error(&format!(
349                "called `Result::unwrap()` on an `Err` value: {message}"
350            )));
351        }
352    }
353
354    #[test]
355    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
356        let classified = capture_operation::<()>(|| panic!("{INCOMPLETE}"))
357            .expect_err("operation panic must be captured");
358
359        assert_eq!(classified.message(), INCOMPLETE);
360        assert!(classified.is_transport());
361        let non_string = capture_operation::<()>(|| std::panic::panic_any(7_u8))
362            .expect_err("non-string panic must be captured");
363        assert_eq!(non_string.message(), "non-string panic payload");
364        assert!(!non_string.is_transport());
365    }
366
367    #[test]
368    fn public_classifier_inspects_the_error_source_chain() {
369        let dead = WrapperError(std::io::Error::other(REFUSED));
370        assert!(is_dead_pocket_ic_transport_error(&dead));
371
372        let unrelated = WrapperError(std::io::Error::other("request rejected"));
373        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
374    }
375
376    #[test]
377    fn snapshot_and_install_wrappers_preserve_transport_causes() {
378        let canister_id = ic_platform::principal::ExternalPrincipal::anonymous();
379        for message in [REFUSED, RESET, "unrelated application panic"] {
380            let expected = message != "unrelated application panic";
381            let capture = ControllerSnapshotError::CaptureFailed {
382                canister_id,
383                sender: None,
384                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
385                    message,
386                ))),
387                cleanup_failures: vec![],
388            };
389            let restore = ControllerSnapshotError::RestoreFailed {
390                canister_id,
391                sender: None,
392                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
393                    message,
394                ))),
395            };
396            for snapshot in [capture, restore] {
397                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
398                let wrapper = WrapperError(std::io::Error::other(snapshot));
399                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
400            }
401            for install in [
402                CanisterInstallError::CreateCanister {
403                    label: Some("fixture".into()),
404                    source: PocketIcOperationError::new(message),
405                },
406                CanisterInstallError::AddCycles {
407                    canister_id,
408                    label: Some("fixture".into()),
409                    source: PocketIcOperationError::new(message),
410                },
411                CanisterInstallError::InstallCode {
412                    canister_id,
413                    label: Some("fixture".into()),
414                    source: PocketIcOperationError::new(message),
415                },
416            ] {
417                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
418                let wrapper = WrapperError(std::io::Error::other(install));
419                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
420            }
421        }
422    }
423
424    #[test]
425    fn unrelated_and_quoted_transport_text_does_not_qualify() {
426        for message in [
427            "application worker channel closed",
428            "fixture expected ConnectionRefused but observed another result",
429            "tcp connect error",
430            "connection closed before message completed",
431            "IncompleteMessage",
432            &format!("fixture quoted {REFUSED}"),
433            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
434            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
435            &INCOMPLETE.replace(
436                "hyper::Error(IncompleteMessage)",
437                "Custom(\"channel closed\")",
438            ),
439            &INCOMPLETE.replace(
440                "hyper::Error(IncompleteMessage)",
441                "Custom(\"hyper::Error(IncompleteMessage)\")",
442            ),
443            &format!("fixture quoted {RESET}"),
444            &RESET.replace("/instances/0/", "/application/worker/"),
445            &RESET.replace("/instances/0/", "/instances/quoted/"),
446            &RESET.replace("kind: ConnectionReset,", "kind: ConnectionAborted,"),
447            &RESET.replace("code: 54", "code: quoted"),
448            &RESET.replace("hyper::Error(Io, Os {", "Custom(Os {"),
449        ] {
450            let error = WrapperError(std::io::Error::other(message.to_owned()));
451            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
452            assert!(!PocketIcOperationError::new(message).is_transport());
453        }
454        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
455        assert!(!is_dead_pocket_ic_transport_error(&refused));
456        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
457        let reset = std::io::Error::from(std::io::ErrorKind::ConnectionReset);
458        assert!(!is_dead_pocket_ic_transport_error(&reset));
459        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(reset)));
460    }
461
462    #[test]
463    fn structured_channel_closure_on_an_instance_request_qualifies() {
464        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
465        assert!(is_dead_instance_transport_error(&message));
466    }
467
468    #[cfg(feature = "wire")]
469    #[test]
470    fn contextual_call_errors_use_their_structured_kind() {
471        use super::super::{WireCallContext, WireCallError};
472        let context = WireCallContext {
473            operation: "query_call",
474            canister_id: ic_platform::principal::ExternalPrincipal::anonymous(),
475            caller: ic_platform::principal::ExternalPrincipal::anonymous(),
476            method: REFUSED,
477            method_id: 1,
478        };
479        for (error, expected) in [
480            (
481                WireCallError::Call {
482                    context: Box::new(context.clone()),
483                    source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
484                        REFUSED,
485                    ))),
486                },
487                true,
488            ),
489            (
490                WireCallError::Decode {
491                    context: Box::new(context.clone()),
492                    bytes: 0,
493                    source: ic_wire::Error::Malformed,
494                },
495                false,
496            ),
497            (
498                WireCallError::Encode {
499                    context: Box::new(context.clone()),
500                    source: ic_wire::Error::Limit,
501                },
502                false,
503            ),
504            (
505                WireCallError::Call {
506                    context: Box::new(context),
507                    source: Box::new(PocketIcCallFailure::Rejected(
508                        ic_platform::pocket_ic::RejectResponse {
509                            reject_code: ic_platform::pocket_ic::RejectCode::CanisterReject,
510                            reject_message: REFUSED.into(),
511                            error_code: ic_platform::pocket_ic::ErrorCode::CanisterRejectedMessage,
512                            certified: false,
513                        },
514                    )),
515                },
516                false,
517            ),
518        ] {
519            assert_eq!(is_dead_pocket_ic_transport_error(&error), expected);
520            let wrapped = WrapperError(std::io::Error::other(std::io::Error::other(error)));
521            assert_eq!(is_dead_pocket_ic_transport_error(&wrapped), expected);
522        }
523    }
524}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
