ic_testkit/pic/
transport.rs

1use std::{
2    any::Any,
3    panic::{AssertUnwindSafe, catch_unwind, resume_unwind},
4};
5
6use ic_platform::pocket_ic::RejectResponse;
7
8/// Captured PocketIC operation failure, retaining transport classification.
9#[derive(Clone, Debug, Eq, PartialEq)]
10pub struct PocketIcOperationError {
11    message: String,
12    transport: bool,
13}
14
15impl PocketIcOperationError {
16    /// Capture an upstream PocketIC panic message and classify its transport.
17    #[must_use]
18    pub fn new(message: impl Into<String>) -> Self {
19        let message = message.into();
20        let transport = is_dead_instance_transport_error(&message);
21        Self { message, transport }
22    }
23
24    /// Read the original upstream message without contextual wrappers.
25    #[must_use]
26    pub fn message(&self) -> &str {
27        &self.message
28    }
29
30    /// Whether the message matches a recognized PocketIC transport failure.
31    #[must_use]
32    pub const fn is_transport(&self) -> bool {
33        self.transport
34    }
35}
36
37// Only callers that intentionally capture every unwind use this boundary.
38// Typed calls retain their separate policy of resuming unrelated panics.
39pub(super) fn capture_operation<T>(
40    operation: impl FnOnce() -> T,
41) -> Result<T, PocketIcOperationError> {
42    catch_unwind(AssertUnwindSafe(operation))
43        .map_err(|payload| PocketIcOperationError::new(panic_payload_to_string(payload.as_ref())))
44}
45
46impl std::fmt::Display for PocketIcOperationError {
47    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48        formatter.write_str(&self.message)
49    }
50}
51
52impl std::error::Error for PocketIcOperationError {}
53
54/// Execute one synchronous PocketIC operation, returning recognized transport panics.
55///
56/// Successful values are returned unchanged, including any upstream `Result` and
57/// its original rejection. Unrelated panics resume with their original payload.
58/// The closure is invoked once: this helper never retries or replays an operation.
59/// A transport failure does not establish whether an update was accepted or
60/// whether an instance died; callers retain recovery and reconciliation policy.
61///
62/// Recognition uses the maintained panic shapes of the selected PocketIC client.
63/// It cannot recover the typed cause discarded upstream, suppress the panic hook,
64/// bound a blocked request, or catch errors while polling a returned future.
65///
66/// ```no_run
67/// use ic_testkit::pic::{PocketIc, PocketIcOperationError, try_pocket_ic_operation};
68///
69/// fn tick(pic: &PocketIc) -> Result<(), PocketIcOperationError> {
70///     try_pocket_ic_operation(|| pic.tick())
71/// }
72/// ```
73///
74/// # Panics
75/// Unrecognized panic payloads continue unwinding.
76pub fn try_pocket_ic_operation<T>(
77    operation: impl FnOnce() -> T,
78) -> Result<T, PocketIcOperationError> {
79    match catch_unwind(AssertUnwindSafe(operation)) {
80        Ok(value) => Ok(value),
81        Err(payload) => {
82            let source = PocketIcOperationError::new(panic_payload_to_string(payload.as_ref()));
83            if source.is_transport() {
84                Err(source)
85            } else {
86                resume_unwind(payload)
87            }
88        }
89    }
90}
91
92/// A rejected management call or a captured PocketIC operation panic.
93#[non_exhaustive]
94#[derive(Debug)]
95pub enum PocketIcCallFailure {
96    /// PocketIC returned a structured management-canister rejection.
97    Rejected(RejectResponse),
98    /// Captured PocketIC panic, retaining its original transport classification.
99    Operation(PocketIcOperationError),
100}
101
102impl std::fmt::Display for PocketIcCallFailure {
103    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
104        match self {
105            Self::Rejected(response) => write!(formatter, "rejected: {response:?}"),
106            Self::Operation(source) if source.is_transport() => {
107                write!(formatter, "PocketIC instance unavailable: {source}")
108            }
109            Self::Operation(source) => write!(formatter, "panicked: {source}"),
110        }
111    }
112}
113
114impl std::error::Error for PocketIcCallFailure {
115    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
116        match self {
117            Self::Rejected(_) => None,
118            Self::Operation(source) => Some(source),
119        }
120    }
121}
122
123pub(super) fn capture_call<T>(
124    call: impl FnOnce() -> Result<T, RejectResponse>,
125) -> Result<T, PocketIcCallFailure> {
126    match capture_operation(call) {
127        Ok(Ok(value)) => Ok(value),
128        Ok(Err(response)) => Err(PocketIcCallFailure::Rejected(response)),
129        Err(source) => Err(PocketIcCallFailure::Operation(source)),
130    }
131}
132
133// Typed codec adapters share one transport policy: retain structured rejection,
134// classify known transport unwinds and resume every unrelated original payload.
135pub(super) fn call_or_resume<T>(
136    call: impl FnOnce() -> Result<T, RejectResponse>,
137) -> Result<T, PocketIcCallFailure> {
138    match try_pocket_ic_operation(call) {
139        Ok(Ok(value)) => Ok(value),
140        Ok(Err(response)) => Err(PocketIcCallFailure::Rejected(response)),
141        Err(source) => Err(PocketIcCallFailure::Operation(source)),
142    }
143}
144
145// Extract a stable string message from one panic payload.
146pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
147    if let Some(message) = payload.downcast_ref::<String>() {
148        return message.clone();
149    }
150    if let Some(message) = payload.downcast_ref::<&'static str>() {
151        return (*message).to_string();
152    }
153
154    "non-string panic payload".to_string()
155}
156
157/// Recognize PocketIC transport failures in typed causes and request-error text.
158///
159/// The complete error source chain is inspected so a recipe can classify its
160/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
161/// preserves the classification already captured by testkit's structured errors.
162/// Testkit [`super::CandidCallError`], [`super::PocketIcCallFailure`], and
163/// [`PocketIcOperationError`] causes retain their transport classification through
164/// contextual display wrappers, including nested [`std::io::Error`] values.
165/// Snapshot and installation failures expose their operation cause through
166/// the error source chain.
167///
168/// For other errors, message recognition requires a reqwest debug error with a
169/// PocketIC instance URL and a recognized transport source, optionally prefixed
170/// by PocketIC's HTTP panic context. Generic application messages and bare I/O
171/// error kinds do not qualify.
172/// Recognized sources include OS connection resets during an HTTP request.
173///
174/// Use this only for errors originating in a PocketIC operation. This is a
175/// message-based heuristic, not proof that the server or instance has died;
176/// another service or application can reproduce the same text. PocketIC does
177/// not yet expose a structured transport error.
178#[must_use]
179pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
180    let mut current = Some(error);
181    while let Some(candidate) = current {
182        // Decoder diagnostics may quote arbitrary reply content. Reaching a
183        // typed decode failure proves this branch is not a transport failure.
184        if candidate.is::<ic_platform::candid::DecodeError>() {
185            return false;
186        }
187        if candidate
188            .downcast_ref::<PocketIcOperationError>()
189            .is_some_and(PocketIcOperationError::is_transport)
190        {
191            return true;
192        }
193        if is_dead_instance_transport_error(&candidate.to_string()) {
194            return true;
195        }
196        // io::Error::source delegates to the contained error's source, skipping
197        // the contained error itself. Inspect it before following its causes so
198        // contextual errors keep their own structured classification.
199        current = if let Some(inner) = candidate
200            .downcast_ref::<std::io::Error>()
201            .and_then(std::io::Error::get_ref)
202        {
203            Some(inner)
204        } else {
205            candidate.source()
206        };
207    }
208    false
209}
210
211// Recognize maintained PocketIC request-error shapes for restore recovery.
212pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
213    let message = message
214        .strip_prefix("HTTP failure: ")
215        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
216        .unwrap_or(message);
217    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
218        return false;
219    };
220    let Some((kind, request)) = error.split_once(", url: \"") else {
221        return false;
222    };
223    if !matches!(kind, "Request" | "Body" | "Decode") {
224        return false;
225    }
226    let Some((url, source)) = request.split_once("\", source: ") else {
227        return false;
228    };
229    let Some(source) = source.strip_suffix(" }") else {
230        return false;
231    };
232    let Some(authority_and_path) = url
233        .strip_prefix("http://")
234        .or_else(|| url.strip_prefix("https://"))
235    else {
236        return false;
237    };
238    let Some((authority, path)) = authority_and_path.split_once('/') else {
239        return false;
240    };
241    let Some(instance_path) = path.strip_prefix("instances/") else {
242        return false;
243    };
244    let instance_id = instance_path.split('/').next().unwrap_or_default();
245    if authority.is_empty()
246        || instance_id.is_empty()
247        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
248    {
249        return false;
250    }
251    let source = source
252        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
253        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
254        .and_then(|source| source.strip_suffix(')'))
255        .unwrap_or(source);
256    let reset = source
257        .strip_prefix("hyper::Error(Io, Os { code: ")
258        .and_then(|source| source.split_once(", kind: ConnectionReset, message: \""))
259        .is_some_and(|(code, message)| code.parse::<i32>().is_ok() && message.ends_with("\" })"));
260    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
261        || reset
262        || matches!(
263            source,
264            "hyper::Error(IncompleteMessage)"
265                | "hyper::Error(ChannelClosed)"
266                | "hyper::Error(Canceled, \"connection closed before message completed\")"
267        )
268}
269
270#[cfg(test)]
271mod tests {
272    use super::super::{CanisterInstallError, ControllerSnapshotError};
273    use super::{
274        PocketIcCallFailure, PocketIcOperationError, capture_operation,
275        is_dead_instance_transport_error, is_dead_pocket_ic_transport_error,
276        try_pocket_ic_operation,
277    };
278
279    #[derive(Debug)]
280    struct WrapperError(std::io::Error);
281
282    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
283    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
284    const RESET: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:49238/instances/0/update/submit_ingress_message\", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(Io, Os { code: 54, kind: ConnectionReset, message: \"Connection reset by peer\" })) }";
285
286    #[test]
287    fn raw_operations_preserve_values_and_rejections() {
288        assert_eq!(try_pocket_ic_operation(|| 42), Ok(42));
289        let rejection = ic_platform::pocket_ic::RejectResponse {
290            reject_code: ic_platform::pocket_ic::RejectCode::CanisterReject,
291            reject_message: "application rejection".into(),
292            error_code: ic_platform::pocket_ic::ErrorCode::CanisterRejectedMessage,
293            certified: false,
294        };
295        let result = try_pocket_ic_operation(|| Err::<(), _>(rejection))
296            .expect("returned rejection is not a transport panic");
297        let rejection = result.expect_err("retain the original rejection");
298        assert_eq!(rejection.reject_message, "application rejection");
299        assert_eq!(
300            rejection.reject_code,
301            ic_platform::pocket_ic::RejectCode::CanisterReject
302        );
303    }
304
305    #[test]
306    fn raw_transport_failure_is_returned_after_one_attempt() {
307        let mut attempts = 0;
308        let error = try_pocket_ic_operation::<()>(|| {
309            attempts += 1;
310            panic!("{REFUSED}");
311        })
312        .expect_err("transport panic must return an error");
313        assert_eq!(attempts, 1);
314        assert_eq!(error.message(), REFUSED);
315        assert!(error.is_transport());
316    }
317
318    impl std::fmt::Display for WrapperError {
319        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
320            formatter.write_str("wrapped PocketIC request failed")
321        }
322    }
323
324    impl std::error::Error for WrapperError {
325        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
326            Some(&self.0)
327        }
328    }
329
330    #[test]
331    fn dead_instance_transport_error_detects_connection_refused() {
332        assert!(is_dead_instance_transport_error(REFUSED));
333        assert!(is_dead_instance_transport_error(&format!(
334            "HTTP failure: {REFUSED}"
335        )));
336    }
337
338    #[test]
339    fn dead_instance_transport_error_detects_incomplete_message() {
340        assert!(is_dead_instance_transport_error(INCOMPLETE));
341        assert!(is_dead_instance_transport_error(&format!(
342            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
343        )));
344    }
345
346    #[test]
347    fn dead_instance_transport_error_detects_native_connection_reset() {
348        for message in [RESET.to_owned(), RESET.replace("code: 54", "code: 104")] {
349            assert!(is_dead_instance_transport_error(&message));
350            assert!(PocketIcOperationError::new(format!("HTTP failure: {message}")).is_transport());
351            assert!(is_dead_instance_transport_error(&format!(
352                "called `Result::unwrap()` on an `Err` value: {message}"
353            )));
354        }
355    }
356
357    #[test]
358    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
359        let classified = capture_operation::<()>(|| panic!("{INCOMPLETE}"))
360            .expect_err("operation panic must be captured");
361
362        assert_eq!(classified.message(), INCOMPLETE);
363        assert!(classified.is_transport());
364        let non_string = capture_operation::<()>(|| std::panic::panic_any(7_u8))
365            .expect_err("non-string panic must be captured");
366        assert_eq!(non_string.message(), "non-string panic payload");
367        assert!(!non_string.is_transport());
368    }
369
370    #[test]
371    fn public_classifier_inspects_the_error_source_chain() {
372        let dead = WrapperError(std::io::Error::other(REFUSED));
373        assert!(is_dead_pocket_ic_transport_error(&dead));
374
375        let unrelated = WrapperError(std::io::Error::other("request rejected"));
376        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
377    }
378
379    #[test]
380    fn snapshot_and_install_wrappers_preserve_transport_causes() {
381        let canister_id = ic_platform::principal::ExternalPrincipal::anonymous();
382        for message in [REFUSED, RESET, "unrelated application panic"] {
383            let expected = message != "unrelated application panic";
384            let capture = ControllerSnapshotError::CaptureFailed {
385                canister_id,
386                sender: None,
387                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
388                    message,
389                ))),
390                cleanup_failures: vec![],
391            };
392            let restore = ControllerSnapshotError::RestoreFailed {
393                canister_id,
394                sender: None,
395                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
396                    message,
397                ))),
398            };
399            for snapshot in [capture, restore] {
400                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
401                let wrapper = WrapperError(std::io::Error::other(snapshot));
402                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
403            }
404            for install in [
405                CanisterInstallError::CreateCanister {
406                    label: Some("fixture".into()),
407                    source: PocketIcOperationError::new(message),
408                },
409                CanisterInstallError::AddCycles {
410                    canister_id,
411                    label: Some("fixture".into()),
412                    source: PocketIcOperationError::new(message),
413                },
414                CanisterInstallError::InstallCode {
415                    canister_id,
416                    label: Some("fixture".into()),
417                    source: PocketIcOperationError::new(message),
418                },
419            ] {
420                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
421                let wrapper = WrapperError(std::io::Error::other(install));
422                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
423            }
424        }
425    }
426
427    #[test]
428    fn unrelated_and_quoted_transport_text_does_not_qualify() {
429        for message in [
430            "application worker channel closed",
431            "fixture expected ConnectionRefused but observed another result",
432            "tcp connect error",
433            "connection closed before message completed",
434            "IncompleteMessage",
435            &format!("fixture quoted {REFUSED}"),
436            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
437            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
438            &INCOMPLETE.replace(
439                "hyper::Error(IncompleteMessage)",
440                "Custom(\"channel closed\")",
441            ),
442            &INCOMPLETE.replace(
443                "hyper::Error(IncompleteMessage)",
444                "Custom(\"hyper::Error(IncompleteMessage)\")",
445            ),
446            &format!("fixture quoted {RESET}"),
447            &RESET.replace("/instances/0/", "/application/worker/"),
448            &RESET.replace("/instances/0/", "/instances/quoted/"),
449            &RESET.replace("kind: ConnectionReset,", "kind: ConnectionAborted,"),
450            &RESET.replace("code: 54", "code: quoted"),
451            &RESET.replace("hyper::Error(Io, Os {", "Custom(Os {"),
452        ] {
453            let error = WrapperError(std::io::Error::other(message.to_owned()));
454            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
455            assert!(!PocketIcOperationError::new(message).is_transport());
456        }
457        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
458        assert!(!is_dead_pocket_ic_transport_error(&refused));
459        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
460        let reset = std::io::Error::from(std::io::ErrorKind::ConnectionReset);
461        assert!(!is_dead_pocket_ic_transport_error(&reset));
462        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(reset)));
463    }
464
465    #[test]
466    fn structured_channel_closure_on_an_instance_request_qualifies() {
467        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
468        assert!(is_dead_instance_transport_error(&message));
469    }
470
471    #[test]
472    fn contextual_call_errors_use_their_structured_kind() {
473        let context = super::super::CandidCallContext::new(
474            "query_call",
475            ic_platform::principal::ExternalPrincipal::anonymous(),
476            ic_platform::principal::ExternalPrincipal::anonymous(),
477            "get",
478        );
479        let error = super::super::CandidCallError::Transport {
480            context: Box::new(context.clone()),
481            source: PocketIcOperationError::new(REFUSED),
482        };
483        assert!(is_dead_pocket_ic_transport_error(&error));
484        let error = super::super::CandidCallError::Decode {
485            context: Box::new(context),
486            bytes: 0,
487            source: Box::new(ic_platform::candid::DecodeError::Candid(
488                ic_platform::candid::contract::Error::msg("channel closed"),
489            )),
490        };
491        assert!(!is_dead_pocket_ic_transport_error(&error));
492    }
493
494    #[test]
495    fn io_wrappers_preserve_contextual_call_error_kinds() {
496        let context = super::super::CandidCallContext::new(
497            "query_call",
498            ic_platform::principal::ExternalPrincipal::anonymous(),
499            ic_platform::principal::ExternalPrincipal::anonymous(),
500            "get",
501        );
502        for (error, expected) in [
503            (
504                super::super::CandidCallError::Transport {
505                    context: Box::new(context.clone()),
506                    source: PocketIcOperationError::new(REFUSED),
507                },
508                true,
509            ),
510            (
511                super::super::CandidCallError::Decode {
512                    context: Box::new(context),
513                    bytes: 0,
514                    source: Box::new(ic_platform::candid::DecodeError::Candid(
515                        ic_platform::candid::contract::Error::msg(REFUSED),
516                    )),
517                },
518                false,
519            ),
520        ] {
521            let inner = std::io::Error::other(error);
522            assert_eq!(is_dead_pocket_ic_transport_error(&inner), expected);
523            let wrapped = WrapperError(std::io::Error::other(inner));
524            assert_eq!(is_dead_pocket_ic_transport_error(&wrapped), expected);
525        }
526    }
527}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
