ic_testkit/pic/
transport.rs

1use std::{
2    any::Any,
3    panic::{AssertUnwindSafe, catch_unwind, resume_unwind},
4};
5
6use ic_platform::pocket_ic::RejectResponse;
7
8/// Captured `PocketIC` operation failure, retaining transport classification.
9#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)]
10#[error("{message}")]
11pub struct PocketIcOperationError {
12    message: String,
13    transport: bool,
14}
15
16impl PocketIcOperationError {
17    /// Capture an upstream `PocketIC` panic message and classify its transport.
18    #[must_use]
19    pub fn new(message: impl Into<String>) -> Self {
20        let message = message.into();
21        let transport = is_dead_instance_transport_error(&message);
22        Self { message, transport }
23    }
24
25    /// Read the original upstream message without contextual wrappers.
26    #[must_use]
27    pub fn message(&self) -> &str {
28        &self.message
29    }
30
31    /// Whether the message matches a recognized `PocketIC` transport failure.
32    #[must_use]
33    pub const fn is_transport(&self) -> bool {
34        self.transport
35    }
36}
37
38// Only callers that intentionally capture every unwind use this boundary.
39// Typed calls retain their separate policy of resuming unrelated panics.
40pub(super) fn capture_operation<T>(
41    operation: impl FnOnce() -> T,
42) -> Result<T, PocketIcOperationError> {
43    catch_unwind(AssertUnwindSafe(operation))
44        .map_err(|payload| PocketIcOperationError::new(panic_payload_to_string(payload.as_ref())))
45}
46
47/// Execute one synchronous `PocketIC` operation, returning recognized transport panics.
48///
49/// Successful values are returned unchanged, including any upstream `Result` and
50/// its original rejection. Unrelated panics resume with their original payload.
51/// The closure is invoked once: this helper never retries or replays an operation.
52/// A transport failure does not establish whether an update was accepted or
53/// whether an instance died; callers retain recovery and reconciliation policy.
54///
55/// Recognition uses the maintained panic shapes of the selected `PocketIC` client.
56/// It cannot recover the typed cause discarded upstream, suppress the panic hook,
57/// bound a blocked request, or catch errors while polling a returned future.
58///
59/// ```no_run
60/// use ic_platform::pocket_ic::PocketIc;
61/// use ic_testkit::pic::{PocketIcOperationError, try_pocket_ic_operation};
62///
63/// fn tick(pic: &PocketIc) -> Result<(), PocketIcOperationError> {
64///     try_pocket_ic_operation(|| pic.tick())
65/// }
66/// ```
67///
68/// # Panics
69/// Unrecognized panic payloads continue unwinding.
70///
71/// # Errors
72/// Returns recognized upstream operation failures; unrelated panics continue unwinding.
73pub fn try_pocket_ic_operation<T>(
74    operation: impl FnOnce() -> T,
75) -> Result<T, PocketIcOperationError> {
76    match catch_unwind(AssertUnwindSafe(operation)) {
77        Ok(value) => Ok(value),
78        Err(payload) => {
79            let source = PocketIcOperationError::new(panic_payload_to_string(payload.as_ref()));
80            if source.is_transport() {
81                Err(source)
82            } else {
83                resume_unwind(payload)
84            }
85        }
86    }
87}
88
89/// A rejected management call or a captured `PocketIC` operation panic.
90#[non_exhaustive]
91#[derive(Debug)]
92pub enum PocketIcCallFailure {
93    /// `PocketIC` returned a structured management-canister rejection.
94    Rejected(RejectResponse),
95    /// Captured `PocketIC` panic, retaining its original transport classification.
96    Operation(PocketIcOperationError),
97}
98
99impl std::fmt::Display for PocketIcCallFailure {
100    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
101        match self {
102            Self::Rejected(response) => write!(formatter, "rejected: {response:?}"),
103            Self::Operation(source) if source.is_transport() => {
104                write!(formatter, "PocketIC instance unavailable: {source}")
105            }
106            Self::Operation(source) => write!(formatter, "panicked: {source}"),
107        }
108    }
109}
110
111impl std::error::Error for PocketIcCallFailure {
112    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
113        match self {
114            Self::Rejected(_) => None,
115            Self::Operation(source) => Some(source),
116        }
117    }
118}
119
120pub(super) fn capture_call<T>(
121    call: impl FnOnce() -> Result<T, RejectResponse>,
122) -> Result<T, PocketIcCallFailure> {
123    capture_operation(call)
124        .map_err(PocketIcCallFailure::Operation)?
125        .map_err(PocketIcCallFailure::Rejected)
126}
127
128/// Execute one synchronous `PocketIC` call, retaining rejection or transport failure.
129///
130/// Uses [`try_pocket_ic_operation`]'s transport recognition and invokes `call`
131/// exactly once. Structured rejections are returned unchanged. Recovery,
132/// reconciliation and lifecycle policy remain with the caller.
133///
134/// # Panics
135/// Unrecognized panic payloads continue unwinding.
136///
137/// # Errors
138/// Returns the original rejection or a recognized upstream transport panic.
139pub fn try_pocket_ic_call<T>(
140    call: impl FnOnce() -> Result<T, RejectResponse>,
141) -> Result<T, PocketIcCallFailure> {
142    try_pocket_ic_operation(call)
143        .map_err(PocketIcCallFailure::Operation)?
144        .map_err(PocketIcCallFailure::Rejected)
145}
146
147// Extract a stable string message from one panic payload.
148pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
149    if let Some(message) = payload.downcast_ref::<String>() {
150        return message.clone();
151    }
152    if let Some(message) = payload.downcast_ref::<&'static str>() {
153        return (*message).to_string();
154    }
155
156    "non-string panic payload".to_string()
157}
158
159/// Recognize `PocketIC` transport failures in typed causes and request-error text.
160///
161/// The complete error source chain is inspected so a recipe can classify its
162/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
163/// preserves the classification already captured by testkit's structured errors.
164/// Testkit's optional Wire call errors, [`super::PocketIcCallFailure`], and
165/// [`PocketIcOperationError`] causes retain their transport classification through
166/// contextual display wrappers, including nested [`std::io::Error`] values.
167/// Snapshot and installation failures expose their operation cause through
168/// the error source chain.
169///
170/// For other errors, message recognition requires a reqwest debug error with a
171/// `PocketIC` instance URL and a recognized transport source, optionally prefixed
172/// by `PocketIC`'s HTTP panic context. Generic application messages and bare I/O
173/// error kinds do not qualify.
174/// Recognized sources include OS connection resets during an HTTP request.
175///
176/// Use this only for errors originating in a `PocketIC` operation. This is a
177/// message-based heuristic, not proof that the server or instance has died;
178/// another service or application can reproduce the same text. `PocketIC` does
179/// not yet expose a structured transport error.
180#[must_use]
181pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
182    let mut current = Some(error);
183    while let Some(candidate) = current {
184        // Owned codec errors and canister rejections are not transport failures.
185        #[cfg(feature = "wire")]
186        if let Some(error) = candidate.downcast_ref::<super::WireCallError>() {
187            return match error {
188                super::WireCallError::Encode { .. } | super::WireCallError::Decode { .. } => false,
189                super::WireCallError::Call { source, .. } => matches!(
190                    source.as_ref(),
191                    PocketIcCallFailure::Operation(error) if error.is_transport()
192                ),
193            };
194        }
195        if let Some(failure) = candidate.downcast_ref::<PocketIcCallFailure>() {
196            return matches!(failure, PocketIcCallFailure::Operation(error) if error.is_transport());
197        }
198        if candidate
199            .downcast_ref::<PocketIcOperationError>()
200            .is_some_and(PocketIcOperationError::is_transport)
201        {
202            return true;
203        }
204        if is_dead_instance_transport_error(&candidate.to_string()) {
205            return true;
206        }
207        // io::Error::source delegates to the contained error's source, skipping
208        // the contained error itself. Inspect it before following its causes so
209        // contextual errors keep their own structured classification.
210        current = candidate
211            .downcast_ref::<std::io::Error>()
212            .and_then(std::io::Error::get_ref)
213            .map(|inner| inner as &dyn std::error::Error)
214            .or_else(|| candidate.source());
215    }
216    false
217}
218
219// Recognize maintained PocketIC request-error shapes for restore recovery.
220pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
221    let message = message
222        .strip_prefix("HTTP failure: ")
223        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
224        .unwrap_or(message);
225    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
226        return false;
227    };
228    let Some((kind, request)) = error.split_once(", url: \"") else {
229        return false;
230    };
231    if !matches!(kind, "Request" | "Body" | "Decode") {
232        return false;
233    }
234    let Some((url, source)) = request.split_once("\", source: ") else {
235        return false;
236    };
237    let Some(source) = source.strip_suffix(" }") else {
238        return false;
239    };
240    let Some(authority_and_path) = url
241        .strip_prefix("http://")
242        .or_else(|| url.strip_prefix("https://"))
243    else {
244        return false;
245    };
246    let Some((authority, path)) = authority_and_path.split_once('/') else {
247        return false;
248    };
249    let Some(instance_path) = path.strip_prefix("instances/") else {
250        return false;
251    };
252    let instance_id = instance_path.split('/').next().unwrap_or_default();
253    if authority.is_empty()
254        || instance_id.is_empty()
255        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
256    {
257        return false;
258    }
259    let source = source
260        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
261        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
262        .and_then(|source| source.strip_suffix(')'))
263        .unwrap_or(source);
264    let reset = source
265        .strip_prefix("hyper::Error(Io, Os { code: ")
266        .and_then(|source| source.split_once(", kind: ConnectionReset, message: \""))
267        .is_some_and(|(code, message)| code.parse::<i32>().is_ok() && message.ends_with("\" })"));
268    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
269        || reset
270        || matches!(
271            source,
272            "hyper::Error(IncompleteMessage)"
273                | "hyper::Error(ChannelClosed)"
274                | "hyper::Error(Canceled, \"connection closed before message completed\")"
275        )
276}
277
278#[cfg(test)]
279mod tests {
280    use super::super::{CanisterInstallError, ControllerSnapshotError};
281    use super::{
282        PocketIcCallFailure, PocketIcOperationError, capture_operation,
283        is_dead_instance_transport_error, is_dead_pocket_ic_transport_error,
284        try_pocket_ic_operation,
285    };
286
287    #[test]
288    fn typed_call_preserves_values_rejections_and_panic_policy_without_replay() {
289        use ic_platform::pocket_ic::{ErrorCode, RejectCode, RejectResponse};
290        use std::{panic::catch_unwind, sync::Arc};
291
292        assert_eq!(super::try_pocket_ic_call(|| Ok(17)).unwrap(), 17);
293        let rejection = RejectResponse {
294            reject_code: RejectCode::CanisterReject,
295            reject_message: "exact rejection".into(),
296            error_code: ErrorCode::CanisterRejectedMessage,
297            certified: false,
298        };
299        let Err(PocketIcCallFailure::Rejected(observed)) =
300            super::try_pocket_ic_call(|| Err::<(), _>(rejection.clone()))
301        else {
302            panic!("expected the structured rejection");
303        };
304        assert_eq!(observed, rejection);
305        let mut attempts = 0;
306        let Err(PocketIcCallFailure::Operation(source)) = super::try_pocket_ic_call::<()>(|| {
307            attempts += 1;
308            panic!("{REFUSED}");
309        }) else {
310            panic!("expected the recognized transport failure");
311        };
312        assert_eq!(attempts, 1);
313        assert!(source.is_transport());
314        assert_eq!(source.message(), REFUSED);
315
316        let payload = Arc::new(17_u8);
317        let resumed = catch_unwind(|| {
318            let _ = super::try_pocket_ic_call::<()>(|| {
319                std::panic::panic_any(Arc::clone(&payload));
320            });
321        })
322        .unwrap_err()
323        .downcast::<Arc<u8>>()
324        .unwrap();
325        assert!(Arc::ptr_eq(&payload, &resumed));
326    }
327
328    #[derive(Debug)]
329    struct WrapperError(std::io::Error);
330
331    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
332    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
333    const RESET: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:49238/instances/0/update/submit_ingress_message\", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(Io, Os { code: 54, kind: ConnectionReset, message: \"Connection reset by peer\" })) }";
334
335    #[test]
336    fn raw_operations_preserve_values_and_rejections() {
337        assert_eq!(try_pocket_ic_operation(|| 42), Ok(42));
338        let rejection = ic_platform::pocket_ic::RejectResponse {
339            reject_code: ic_platform::pocket_ic::RejectCode::CanisterReject,
340            reject_message: "application rejection".into(),
341            error_code: ic_platform::pocket_ic::ErrorCode::CanisterRejectedMessage,
342            certified: false,
343        };
344        let result = try_pocket_ic_operation(|| Err::<(), _>(rejection))
345            .expect("returned rejection is not a transport panic");
346        let rejection = result.expect_err("retain the original rejection");
347        assert_eq!(rejection.reject_message, "application rejection");
348        assert_eq!(
349            rejection.reject_code,
350            ic_platform::pocket_ic::RejectCode::CanisterReject
351        );
352    }
353
354    #[test]
355    fn raw_transport_failure_is_returned_after_one_attempt() {
356        let mut attempts = 0;
357        let error = try_pocket_ic_operation::<()>(|| {
358            attempts += 1;
359            panic!("{REFUSED}");
360        })
361        .expect_err("transport panic must return an error");
362        assert_eq!(attempts, 1);
363        assert_eq!(error.message(), REFUSED);
364        assert!(error.is_transport());
365    }
366
367    impl std::fmt::Display for WrapperError {
368        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
369            formatter.write_str("wrapped PocketIC request failed")
370        }
371    }
372
373    impl std::error::Error for WrapperError {
374        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
375            Some(&self.0)
376        }
377    }
378
379    #[test]
380    fn dead_instance_transport_error_detects_connection_refused() {
381        assert!(is_dead_instance_transport_error(REFUSED));
382        assert!(is_dead_instance_transport_error(&format!(
383            "HTTP failure: {REFUSED}"
384        )));
385    }
386
387    #[test]
388    fn dead_instance_transport_error_detects_incomplete_message() {
389        assert!(is_dead_instance_transport_error(INCOMPLETE));
390        assert!(is_dead_instance_transport_error(&format!(
391            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
392        )));
393    }
394
395    #[test]
396    fn dead_instance_transport_error_detects_native_connection_reset() {
397        for message in [RESET.to_owned(), RESET.replace("code: 54", "code: 104")] {
398            assert!(is_dead_instance_transport_error(&message));
399            assert!(PocketIcOperationError::new(format!("HTTP failure: {message}")).is_transport());
400            assert!(is_dead_instance_transport_error(&format!(
401                "called `Result::unwrap()` on an `Err` value: {message}"
402            )));
403        }
404    }
405
406    #[test]
407    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
408        let classified = capture_operation::<()>(|| panic!("{INCOMPLETE}"))
409            .expect_err("operation panic must be captured");
410
411        assert_eq!(classified.message(), INCOMPLETE);
412        assert!(classified.is_transport());
413        let non_string = capture_operation::<()>(|| std::panic::panic_any(7_u8))
414            .expect_err("non-string panic must be captured");
415        assert_eq!(non_string.message(), "non-string panic payload");
416        assert!(!non_string.is_transport());
417    }
418
419    #[test]
420    fn public_classifier_inspects_the_error_source_chain() {
421        let dead = WrapperError(std::io::Error::other(REFUSED));
422        assert!(is_dead_pocket_ic_transport_error(&dead));
423
424        let unrelated = WrapperError(std::io::Error::other("request rejected"));
425        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
426    }
427
428    #[test]
429    fn snapshot_and_install_wrappers_preserve_transport_causes() {
430        let canister_id = ic_platform::principal::ExternalPrincipal::anonymous();
431        for message in [REFUSED, RESET, "unrelated application panic"] {
432            let expected = message != "unrelated application panic";
433            let capture = ControllerSnapshotError::CaptureFailed {
434                canister_id,
435                sender: None,
436                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
437                    message,
438                ))),
439                cleanup_failures: vec![],
440            };
441            let restore = ControllerSnapshotError::RestoreFailed {
442                canister_id,
443                sender: None,
444                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
445                    message,
446                ))),
447            };
448            for snapshot in [capture, restore] {
449                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
450                let wrapper = WrapperError(std::io::Error::other(snapshot));
451                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
452            }
453            for install in [
454                CanisterInstallError::CreateCanister {
455                    label: Some("fixture".into()),
456                    source: PocketIcOperationError::new(message),
457                },
458                CanisterInstallError::AddCycles {
459                    canister_id,
460                    label: Some("fixture".into()),
461                    source: PocketIcOperationError::new(message),
462                },
463                CanisterInstallError::InstallCode {
464                    canister_id,
465                    label: Some("fixture".into()),
466                    source: PocketIcOperationError::new(message),
467                },
468            ] {
469                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
470                let wrapper = WrapperError(std::io::Error::other(install));
471                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
472            }
473        }
474    }
475
476    #[test]
477    fn unrelated_and_quoted_transport_text_does_not_qualify() {
478        for message in [
479            "application worker channel closed",
480            "fixture expected ConnectionRefused but observed another result",
481            "tcp connect error",
482            "connection closed before message completed",
483            "IncompleteMessage",
484            &format!("fixture quoted {REFUSED}"),
485            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
486            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
487            &INCOMPLETE.replace(
488                "hyper::Error(IncompleteMessage)",
489                "Custom(\"channel closed\")",
490            ),
491            &INCOMPLETE.replace(
492                "hyper::Error(IncompleteMessage)",
493                "Custom(\"hyper::Error(IncompleteMessage)\")",
494            ),
495            &format!("fixture quoted {RESET}"),
496            &RESET.replace("/instances/0/", "/application/worker/"),
497            &RESET.replace("/instances/0/", "/instances/quoted/"),
498            &RESET.replace("kind: ConnectionReset,", "kind: ConnectionAborted,"),
499            &RESET.replace("code: 54", "code: quoted"),
500            &RESET.replace("hyper::Error(Io, Os {", "Custom(Os {"),
501        ] {
502            let error = WrapperError(std::io::Error::other(message.to_owned()));
503            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
504            assert!(!PocketIcOperationError::new(message).is_transport());
505        }
506        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
507        assert!(!is_dead_pocket_ic_transport_error(&refused));
508        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
509        let reset = std::io::Error::from(std::io::ErrorKind::ConnectionReset);
510        assert!(!is_dead_pocket_ic_transport_error(&reset));
511        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(reset)));
512    }
513
514    #[test]
515    fn structured_channel_closure_on_an_instance_request_qualifies() {
516        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
517        assert!(is_dead_instance_transport_error(&message));
518    }
519
520    #[cfg(feature = "wire")]
521    #[test]
522    fn contextual_call_errors_use_their_structured_kind() {
523        use super::super::{WireCallContext, WireCallError};
524        let context = WireCallContext {
525            operation: "query_call",
526            canister_id: ic_platform::principal::ExternalPrincipal::anonymous(),
527            caller: ic_platform::principal::ExternalPrincipal::anonymous(),
528            method: REFUSED,
529            method_id: 1,
530        };
531        for (error, expected) in [
532            (
533                WireCallError::Call {
534                    context: Box::new(context.clone()),
535                    source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
536                        REFUSED,
537                    ))),
538                },
539                true,
540            ),
541            (
542                WireCallError::Decode {
543                    context: Box::new(context.clone()),
544                    bytes: 0,
545                    source: ic_wire::Error::Malformed,
546                },
547                false,
548            ),
549            (
550                WireCallError::Encode {
551                    context: Box::new(context.clone()),
552                    source: ic_wire::Error::Limit,
553                },
554                false,
555            ),
556            (
557                WireCallError::Call {
558                    context: Box::new(context),
559                    source: Box::new(PocketIcCallFailure::Rejected(
560                        ic_platform::pocket_ic::RejectResponse {
561                            reject_code: ic_platform::pocket_ic::RejectCode::CanisterReject,
562                            reject_message: REFUSED.into(),
563                            error_code: ic_platform::pocket_ic::ErrorCode::CanisterRejectedMessage,
564                            certified: false,
565                        },
566                    )),
567                },
568                false,
569            ),
570        ] {
571            assert_eq!(is_dead_pocket_ic_transport_error(&error), expected);
572            let wrapped = WrapperError(std::io::Error::other(std::io::Error::other(error)));
573            assert_eq!(is_dead_pocket_ic_transport_error(&wrapped), expected);
574        }
575    }
576}
ic_testkit/pic/transport.rs

ic_testkit/pic/
transport.rs
