ic-testkit 0.76.0

use std::{
    any::Any,
    panic::{AssertUnwindSafe, catch_unwind, resume_unwind},
};

use ic_platform::pocket_ic::RejectResponse;

/// Captured `PocketIC` operation failure, retaining transport classification.
#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)]
#[error("{message}")]
pub struct PocketIcOperationError {
    message: String,
    transport: bool,
}

impl PocketIcOperationError {
    /// Capture an upstream `PocketIC` panic message and classify its transport.
    #[must_use]
    pub fn new(message: impl Into<String>) -> Self {
        let message = message.into();
        let transport = is_dead_instance_transport_error(&message);
        Self { message, transport }
    }

    /// Read the original upstream message without contextual wrappers.
    #[must_use]
    pub fn message(&self) -> &str {
        &self.message
    }

    /// Whether the message matches a recognized `PocketIC` transport failure.
    #[must_use]
    pub const fn is_transport(&self) -> bool {
        self.transport
    }
}

// Only callers that intentionally capture every unwind use this boundary.
// Typed calls retain their separate policy of resuming unrelated panics.
pub(super) fn capture_operation<T>(
    operation: impl FnOnce() -> T,
) -> Result<T, PocketIcOperationError> {
    catch_unwind(AssertUnwindSafe(operation))
        .map_err(|payload| PocketIcOperationError::new(panic_payload_to_string(payload.as_ref())))
}

/// Execute one synchronous `PocketIC` operation, returning recognized transport panics.
///
/// Successful values are returned unchanged, including any upstream `Result` and
/// its original rejection. Unrelated panics resume with their original payload.
/// The closure is invoked once: this helper never retries or replays an operation.
/// A transport failure does not establish whether an update was accepted or
/// whether an instance died; callers retain recovery and reconciliation policy.
///
/// Recognition uses the maintained panic shapes of the selected `PocketIC` client.
/// It cannot recover the typed cause discarded upstream, suppress the panic hook,
/// bound a blocked request, or catch errors while polling a returned future.
///
/// ```no_run
/// use ic_platform::pocket_ic::PocketIc;
/// use ic_testkit::pic::{PocketIcOperationError, try_pocket_ic_operation};
///
/// fn tick(pic: &PocketIc) -> Result<(), PocketIcOperationError> {
///     try_pocket_ic_operation(|| pic.tick())
/// }
/// ```
///
/// # Panics
/// Unrecognized panic payloads continue unwinding.
///
/// # Errors
/// Returns recognized upstream operation failures; unrelated panics continue unwinding.
pub fn try_pocket_ic_operation<T>(
    operation: impl FnOnce() -> T,
) -> Result<T, PocketIcOperationError> {
    match catch_unwind(AssertUnwindSafe(operation)) {
        Ok(value) => Ok(value),
        Err(payload) => {
            let source = PocketIcOperationError::new(panic_payload_to_string(payload.as_ref()));
            if source.is_transport() {
                Err(source)
            } else {
                resume_unwind(payload)
            }
        }
    }
}

/// A rejected management call or a captured `PocketIC` operation panic.
#[non_exhaustive]
#[derive(Debug)]
pub enum PocketIcCallFailure {
    /// `PocketIC` returned a structured management-canister rejection.
    Rejected(RejectResponse),
    /// Captured `PocketIC` panic, retaining its original transport classification.
    Operation(PocketIcOperationError),
}

impl std::fmt::Display for PocketIcCallFailure {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::Rejected(response) => write!(formatter, "rejected: {response:?}"),
            Self::Operation(source) if source.is_transport() => {
                write!(formatter, "PocketIC instance unavailable: {source}")
            }
            Self::Operation(source) => write!(formatter, "panicked: {source}"),
        }
    }
}

impl std::error::Error for PocketIcCallFailure {
    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        match self {
            Self::Rejected(_) => None,
            Self::Operation(source) => Some(source),
        }
    }
}

pub(super) fn capture_call<T>(
    call: impl FnOnce() -> Result<T, RejectResponse>,
) -> Result<T, PocketIcCallFailure> {
    capture_operation(call)
        .map_err(PocketIcCallFailure::Operation)?
        .map_err(PocketIcCallFailure::Rejected)
}

/// Execute one synchronous `PocketIC` call, retaining rejection or transport failure.
///
/// Uses [`try_pocket_ic_operation`]'s transport recognition and invokes `call`
/// exactly once. Structured rejections are returned unchanged. Recovery,
/// reconciliation and lifecycle policy remain with the caller.
///
/// # Panics
/// Unrecognized panic payloads continue unwinding.
///
/// # Errors
/// Returns the original rejection or a recognized upstream transport panic.
pub fn try_pocket_ic_call<T>(
    call: impl FnOnce() -> Result<T, RejectResponse>,
) -> Result<T, PocketIcCallFailure> {
    try_pocket_ic_operation(call)
        .map_err(PocketIcCallFailure::Operation)?
        .map_err(PocketIcCallFailure::Rejected)
}

// Extract a stable string message from one panic payload.
pub(super) fn panic_payload_to_string(payload: &(dyn Any + Send)) -> String {
    if let Some(message) = payload.downcast_ref::<String>() {
        return message.clone();
    }
    if let Some(message) = payload.downcast_ref::<&'static str>() {
        return (*message).to_string();
    }

    "non-string panic payload".to_string()
}

/// Recognize `PocketIC` transport failures in typed causes and request-error text.
///
/// The complete error source chain is inspected so a recipe can classify its
/// own wrapper error as [`super::RebuildReason::DeadPocketIcTransport`]. This
/// preserves the classification already captured by testkit's structured errors.
/// Testkit's optional Wire call errors, [`super::PocketIcCallFailure`], and
/// [`PocketIcOperationError`] causes retain their transport classification through
/// contextual display wrappers, including nested [`std::io::Error`] values.
/// Snapshot and installation failures expose their operation cause through
/// the error source chain.
///
/// For other errors, message recognition requires a reqwest debug error with a
/// `PocketIC` instance URL and a recognized transport source, optionally prefixed
/// by `PocketIC`'s HTTP panic context. Generic application messages and bare I/O
/// error kinds do not qualify.
/// Recognized sources include OS connection resets during an HTTP request.
///
/// Use this only for errors originating in a `PocketIC` operation. This is a
/// message-based heuristic, not proof that the server or instance has died;
/// another service or application can reproduce the same text. `PocketIC` does
/// not yet expose a structured transport error.
#[must_use]
pub fn is_dead_pocket_ic_transport_error(error: &(dyn std::error::Error + 'static)) -> bool {
    let mut current = Some(error);
    while let Some(candidate) = current {
        // Owned codec errors and canister rejections are not transport failures.
        #[cfg(feature = "wire")]
        if let Some(error) = candidate.downcast_ref::<super::WireCallError>() {
            return match error {
                super::WireCallError::Encode { .. } | super::WireCallError::Decode { .. } => false,
                super::WireCallError::Call { source, .. } => matches!(
                    source.as_ref(),
                    PocketIcCallFailure::Operation(error) if error.is_transport()
                ),
            };
        }
        if let Some(failure) = candidate.downcast_ref::<PocketIcCallFailure>() {
            return matches!(failure, PocketIcCallFailure::Operation(error) if error.is_transport());
        }
        if candidate
            .downcast_ref::<PocketIcOperationError>()
            .is_some_and(PocketIcOperationError::is_transport)
        {
            return true;
        }
        if is_dead_instance_transport_error(&candidate.to_string()) {
            return true;
        }
        // io::Error::source delegates to the contained error's source, skipping
        // the contained error itself. Inspect it before following its causes so
        // contextual errors keep their own structured classification.
        current = candidate
            .downcast_ref::<std::io::Error>()
            .and_then(std::io::Error::get_ref)
            .map(|inner| inner as &dyn std::error::Error)
            .or_else(|| candidate.source());
    }
    false
}

// Recognize maintained PocketIC request-error shapes for restore recovery.
pub(super) fn is_dead_instance_transport_error(message: &str) -> bool {
    let message = message
        .strip_prefix("HTTP failure: ")
        .or_else(|| message.strip_prefix("called `Result::unwrap()` on an `Err` value: "))
        .unwrap_or(message);
    let Some(error) = message.strip_prefix("reqwest::Error { kind: ") else {
        return false;
    };
    let Some((kind, request)) = error.split_once(", url: \"") else {
        return false;
    };
    if !matches!(kind, "Request" | "Body" | "Decode") {
        return false;
    }
    let Some((url, source)) = request.split_once("\", source: ") else {
        return false;
    };
    let Some(source) = source.strip_suffix(" }") else {
        return false;
    };
    let Some(authority_and_path) = url
        .strip_prefix("http://")
        .or_else(|| url.strip_prefix("https://"))
    else {
        return false;
    };
    let Some((authority, path)) = authority_and_path.split_once('/') else {
        return false;
    };
    let Some(instance_path) = path.strip_prefix("instances/") else {
        return false;
    };
    let instance_id = instance_path.split('/').next().unwrap_or_default();
    if authority.is_empty()
        || instance_id.is_empty()
        || !instance_id.bytes().all(|byte| byte.is_ascii_digit())
    {
        return false;
    }
    let source = source
        .strip_prefix("hyper_util::client::legacy::Error(Connect, ")
        .or_else(|| source.strip_prefix("hyper_util::client::legacy::Error(SendRequest, "))
        .and_then(|source| source.strip_suffix(')'))
        .unwrap_or(source);
    let reset = source
        .strip_prefix("hyper::Error(Io, Os { code: ")
        .and_then(|source| source.split_once(", kind: ConnectionReset, message: \""))
        .is_some_and(|(code, message)| code.parse::<i32>().is_ok() && message.ends_with("\" })"));
    (source.starts_with("ConnectError(") && source.contains("kind: ConnectionRefused,"))
        || reset
        || matches!(
            source,
            "hyper::Error(IncompleteMessage)"
                | "hyper::Error(ChannelClosed)"
                | "hyper::Error(Canceled, \"connection closed before message completed\")"
        )
}

#[cfg(test)]
mod tests {
    use super::super::{CanisterInstallError, ControllerSnapshotError};
    use super::{
        PocketIcCallFailure, PocketIcOperationError, capture_operation,
        is_dead_instance_transport_error, is_dead_pocket_ic_transport_error,
        try_pocket_ic_operation,
    };

    #[test]
    fn typed_call_preserves_values_rejections_and_panic_policy_without_replay() {
        use ic_platform::pocket_ic::{ErrorCode, RejectCode, RejectResponse};
        use std::{panic::catch_unwind, sync::Arc};

        assert_eq!(super::try_pocket_ic_call(|| Ok(17)).unwrap(), 17);
        let rejection = RejectResponse {
            reject_code: RejectCode::CanisterReject,
            reject_message: "exact rejection".into(),
            error_code: ErrorCode::CanisterRejectedMessage,
            certified: false,
        };
        let Err(PocketIcCallFailure::Rejected(observed)) =
            super::try_pocket_ic_call(|| Err::<(), _>(rejection.clone()))
        else {
            panic!("expected the structured rejection");
        };
        assert_eq!(observed, rejection);
        let mut attempts = 0;
        let Err(PocketIcCallFailure::Operation(source)) = super::try_pocket_ic_call::<()>(|| {
            attempts += 1;
            panic!("{REFUSED}");
        }) else {
            panic!("expected the recognized transport failure");
        };
        assert_eq!(attempts, 1);
        assert!(source.is_transport());
        assert_eq!(source.message(), REFUSED);

        let payload = Arc::new(17_u8);
        let resumed = catch_unwind(|| {
            let _ = super::try_pocket_ic_call::<()>(|| {
                std::panic::panic_any(Arc::clone(&payload));
            });
        })
        .unwrap_err()
        .downcast::<Arc<u8>>()
        .unwrap();
        assert!(Arc::ptr_eq(&payload, &resumed));
    }

    #[derive(Debug)]
    struct WrapperError(std::io::Error);

    const REFUSED: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/update/tick\", source: hyper_util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", 127.0.0.1:1234, Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }";
    const INCOMPLETE: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:1234/instances/0/read/get_time\", source: hyper::Error(IncompleteMessage) }";
    const RESET: &str = "reqwest::Error { kind: Request, url: \"http://127.0.0.1:49238/instances/0/update/submit_ingress_message\", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(Io, Os { code: 54, kind: ConnectionReset, message: \"Connection reset by peer\" })) }";

    #[test]
    fn raw_operations_preserve_values_and_rejections() {
        assert_eq!(try_pocket_ic_operation(|| 42), Ok(42));
        let rejection = ic_platform::pocket_ic::RejectResponse {
            reject_code: ic_platform::pocket_ic::RejectCode::CanisterReject,
            reject_message: "application rejection".into(),
            error_code: ic_platform::pocket_ic::ErrorCode::CanisterRejectedMessage,
            certified: false,
        };
        let result = try_pocket_ic_operation(|| Err::<(), _>(rejection))
            .expect("returned rejection is not a transport panic");
        let rejection = result.expect_err("retain the original rejection");
        assert_eq!(rejection.reject_message, "application rejection");
        assert_eq!(
            rejection.reject_code,
            ic_platform::pocket_ic::RejectCode::CanisterReject
        );
    }

    #[test]
    fn raw_transport_failure_is_returned_after_one_attempt() {
        let mut attempts = 0;
        let error = try_pocket_ic_operation::<()>(|| {
            attempts += 1;
            panic!("{REFUSED}");
        })
        .expect_err("transport panic must return an error");
        assert_eq!(attempts, 1);
        assert_eq!(error.message(), REFUSED);
        assert!(error.is_transport());
    }

    impl std::fmt::Display for WrapperError {
        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            formatter.write_str("wrapped PocketIC request failed")
        }
    }

    impl std::error::Error for WrapperError {
        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
            Some(&self.0)
        }
    }

    #[test]
    fn dead_instance_transport_error_detects_connection_refused() {
        assert!(is_dead_instance_transport_error(REFUSED));
        assert!(is_dead_instance_transport_error(&format!(
            "HTTP failure: {REFUSED}"
        )));
    }

    #[test]
    fn dead_instance_transport_error_detects_incomplete_message() {
        assert!(is_dead_instance_transport_error(INCOMPLETE));
        assert!(is_dead_instance_transport_error(&format!(
            "called `Result::unwrap()` on an `Err` value: {INCOMPLETE}"
        )));
    }

    #[test]
    fn dead_instance_transport_error_detects_native_connection_reset() {
        for message in [RESET.to_owned(), RESET.replace("code: 54", "code: 104")] {
            assert!(is_dead_instance_transport_error(&message));
            assert!(PocketIcOperationError::new(format!("HTTP failure: {message}")).is_transport());
            assert!(is_dead_instance_transport_error(&format!(
                "called `Result::unwrap()` on an `Err` value: {message}"
            )));
        }
    }

    #[test]
    fn classify_pocket_ic_panic_marks_dead_instance_transport() {
        let classified = capture_operation::<()>(|| panic!("{INCOMPLETE}"))
            .expect_err("operation panic must be captured");

        assert_eq!(classified.message(), INCOMPLETE);
        assert!(classified.is_transport());
        let non_string = capture_operation::<()>(|| std::panic::panic_any(7_u8))
            .expect_err("non-string panic must be captured");
        assert_eq!(non_string.message(), "non-string panic payload");
        assert!(!non_string.is_transport());
    }

    #[test]
    fn public_classifier_inspects_the_error_source_chain() {
        let dead = WrapperError(std::io::Error::other(REFUSED));
        assert!(is_dead_pocket_ic_transport_error(&dead));

        let unrelated = WrapperError(std::io::Error::other("request rejected"));
        assert!(!is_dead_pocket_ic_transport_error(&unrelated));
    }

    #[test]
    fn snapshot_and_install_wrappers_preserve_transport_causes() {
        let canister_id = ic_platform::principal::ExternalPrincipal::anonymous();
        for message in [REFUSED, RESET, "unrelated application panic"] {
            let expected = message != "unrelated application panic";
            let capture = ControllerSnapshotError::CaptureFailed {
                canister_id,
                sender: None,
                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
                    message,
                ))),
                cleanup_failures: vec![],
            };
            let restore = ControllerSnapshotError::RestoreFailed {
                canister_id,
                sender: None,
                source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
                    message,
                ))),
            };
            for snapshot in [capture, restore] {
                assert_eq!(is_dead_pocket_ic_transport_error(&snapshot), expected);
                let wrapper = WrapperError(std::io::Error::other(snapshot));
                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
            }
            for install in [
                CanisterInstallError::CreateCanister {
                    label: Some("fixture".into()),
                    source: PocketIcOperationError::new(message),
                },
                CanisterInstallError::AddCycles {
                    canister_id,
                    label: Some("fixture".into()),
                    source: PocketIcOperationError::new(message),
                },
                CanisterInstallError::InstallCode {
                    canister_id,
                    label: Some("fixture".into()),
                    source: PocketIcOperationError::new(message),
                },
            ] {
                assert_eq!(is_dead_pocket_ic_transport_error(&install), expected);
                let wrapper = WrapperError(std::io::Error::other(install));
                assert_eq!(is_dead_pocket_ic_transport_error(&wrapper), expected);
            }
        }
    }

    #[test]
    fn unrelated_and_quoted_transport_text_does_not_qualify() {
        for message in [
            "application worker channel closed",
            "fixture expected ConnectionRefused but observed another result",
            "tcp connect error",
            "connection closed before message completed",
            "IncompleteMessage",
            &format!("fixture quoted {REFUSED}"),
            &REFUSED.replace("/instances/0/update/tick", "/application/worker"),
            &REFUSED.replace("/instances/0/", "/instances/quoted/"),
            &INCOMPLETE.replace(
                "hyper::Error(IncompleteMessage)",
                "Custom(\"channel closed\")",
            ),
            &INCOMPLETE.replace(
                "hyper::Error(IncompleteMessage)",
                "Custom(\"hyper::Error(IncompleteMessage)\")",
            ),
            &format!("fixture quoted {RESET}"),
            &RESET.replace("/instances/0/", "/application/worker/"),
            &RESET.replace("/instances/0/", "/instances/quoted/"),
            &RESET.replace("kind: ConnectionReset,", "kind: ConnectionAborted,"),
            &RESET.replace("code: 54", "code: quoted"),
            &RESET.replace("hyper::Error(Io, Os {", "Custom(Os {"),
        ] {
            let error = WrapperError(std::io::Error::other(message.to_owned()));
            assert!(!is_dead_pocket_ic_transport_error(&error), "{message}");
            assert!(!PocketIcOperationError::new(message).is_transport());
        }
        let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
        assert!(!is_dead_pocket_ic_transport_error(&refused));
        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(refused)));
        let reset = std::io::Error::from(std::io::ErrorKind::ConnectionReset);
        assert!(!is_dead_pocket_ic_transport_error(&reset));
        assert!(!is_dead_pocket_ic_transport_error(&WrapperError(reset)));
    }

    #[test]
    fn structured_channel_closure_on_an_instance_request_qualifies() {
        let message = INCOMPLETE.replace("IncompleteMessage", "ChannelClosed");
        assert!(is_dead_instance_transport_error(&message));
    }

    #[cfg(feature = "wire")]
    #[test]
    fn contextual_call_errors_use_their_structured_kind() {
        use super::super::{WireCallContext, WireCallError};
        let context = WireCallContext {
            operation: "query_call",
            canister_id: ic_platform::principal::ExternalPrincipal::anonymous(),
            caller: ic_platform::principal::ExternalPrincipal::anonymous(),
            method: REFUSED,
            method_id: 1,
        };
        for (error, expected) in [
            (
                WireCallError::Call {
                    context: Box::new(context.clone()),
                    source: Box::new(PocketIcCallFailure::Operation(PocketIcOperationError::new(
                        REFUSED,
                    ))),
                },
                true,
            ),
            (
                WireCallError::Decode {
                    context: Box::new(context.clone()),
                    bytes: 0,
                    source: ic_wire::Error::Malformed,
                },
                false,
            ),
            (
                WireCallError::Encode {
                    context: Box::new(context.clone()),
                    source: ic_wire::Error::Limit,
                },
                false,
            ),
            (
                WireCallError::Call {
                    context: Box::new(context),
                    source: Box::new(PocketIcCallFailure::Rejected(
                        ic_platform::pocket_ic::RejectResponse {
                            reject_code: ic_platform::pocket_ic::RejectCode::CanisterReject,
                            reject_message: REFUSED.into(),
                            error_code: ic_platform::pocket_ic::ErrorCode::CanisterRejectedMessage,
                            certified: false,
                        },
                    )),
                },
                false,
            ),
        ] {
            assert_eq!(is_dead_pocket_ic_transport_error(&error), expected);
            let wrapped = WrapperError(std::io::Error::other(std::io::Error::other(error)));
            assert_eq!(is_dead_pocket_ic_transport_error(&wrapped), expected);
        }
    }
}