#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
#[error("transport error")]
Transport(#[source] Box<tonic::transport::Error>),
#[error("connection error: {0}")]
Connection(String),
#[error("grpc status {}: {}", .0.code(), .0.message())]
Status(#[source] Box<tonic::Status>),
#[error("http {status}: {body}")]
Http {
status: u16,
body: String,
},
#[error("json error: {0}")]
Json(#[source] Box<serde_json::Error>),
#[error("command rejected ({code}): {message}")]
CommandRejected {
code: String,
message: String,
},
#[error("authentication failed: {0}")]
Auth(String),
#[error("invalid request: {0}")]
InvalidRequest(String),
#[error("unexpected response: {0}")]
UnexpectedResponse(String),
#[error("operation timed out")]
Timeout,
#[error("payload conversion failed: {0}")]
Payload(#[source] Box<dyn std::error::Error + Send + Sync>),
}
impl Error {
#[must_use]
pub fn code(&self) -> Option<tonic::Code> {
match self {
Error::Status(status) => Some(status.code()),
Error::Http { body, .. } => http_grpc_code(body),
_ => None,
}
}
#[must_use]
pub fn is_retriable(&self) -> bool {
use tonic::Code::{Aborted, DeadlineExceeded, ResourceExhausted, Unavailable};
match self {
Error::Timeout | Error::Transport(_) | Error::Connection(_) => true,
Error::Status(status) => match status_category(status) {
Some(category) => category.is_retriable(),
None => {
status_retry_delay(status).is_some()
|| matches!(
status.code(),
Unavailable | DeadlineExceeded | ResourceExhausted | Aborted
)
}
},
Error::Http { status, body } => match http_category(body) {
Some(category) => category.is_retriable(),
None => http_retry_delay(body).is_some() || matches!(status, 408 | 429 | 500..=599),
},
_ => false,
}
}
#[must_use]
pub fn category(&self) -> Option<ErrorCategory> {
match self {
Error::Status(status) => status_category(status),
Error::Http { body, .. } => http_category(body),
_ => None,
}
}
#[must_use]
pub fn retry_delay(&self) -> Option<std::time::Duration> {
match self {
Error::Status(status) => status_retry_delay(status),
Error::Http { body, .. } => http_retry_delay(body),
_ => None,
}
}
#[must_use]
pub fn correlation_id(&self) -> Option<String> {
match self {
Error::Status(status) => {
use tonic_types::StatusExt as _;
status
.get_details_request_info()
.map(|info| info.request_id)
}
Error::Http { body, .. } => http_correlation_id(body),
_ => None,
}
}
#[must_use]
pub fn resource_info(&self) -> Vec<ResourceInfo> {
match self {
Error::Status(status) => {
use tonic_types::StatusExt as _;
status
.get_details_resource_info()
.map(|info| ResourceInfo {
resource_type: info.resource_type,
resource_name: info.resource_name,
owner: info.owner,
description: info.description,
})
.into_iter()
.collect()
}
Error::Http { body, .. } => http_resource_info(body),
_ => Vec::new(),
}
}
#[must_use]
pub fn error_info(&self) -> Option<ErrorInfo> {
match self {
Error::Status(status) => {
use tonic_types::StatusExt as _;
status
.get_error_details()
.error_info()
.map(|info| ErrorInfo {
reason: info.reason.clone(),
domain: info.domain.clone(),
metadata: info.metadata.clone(),
})
}
Error::Http { body, .. } => http_error_info(body),
_ => None,
}
}
}
fn status_category(status: &tonic::Status) -> Option<ErrorCategory> {
use tonic_types::StatusExt as _;
let info = status.get_details_error_info()?;
ErrorCategory::from_i32(info.metadata.get("category")?.parse().ok()?)
}
fn status_retry_delay(status: &tonic::Status) -> Option<std::time::Duration> {
use tonic_types::StatusExt as _;
status.get_details_retry_info()?.retry_delay
}
fn http_category(body: &str) -> Option<ErrorCategory> {
let body: serde_json::Value = serde_json::from_str(body).ok()?;
let id = body.get("errorCategory")?.as_i64()?;
ErrorCategory::from_i32(i32::try_from(id).ok()?)
}
fn http_retry_delay(body: &str) -> Option<std::time::Duration> {
let body: serde_json::Value = serde_json::from_str(body).ok()?;
parse_spelled_duration(body.get("retryInfo")?.as_str()?)
}
fn http_resource_info(body: &str) -> Vec<ResourceInfo> {
let Ok(body) = serde_json::from_str::<serde_json::Value>(body) else {
return Vec::new();
};
let Some(resources) = body.get("resources").and_then(serde_json::Value::as_array) else {
return Vec::new();
};
resources
.iter()
.filter_map(|entry| {
let pair = entry.as_array()?;
Some(ResourceInfo {
resource_type: pair.first()?.as_str()?.to_string(),
resource_name: pair.get(1)?.as_str()?.to_string(),
owner: String::new(),
description: String::new(),
})
})
.collect()
}
fn http_grpc_code(body: &str) -> Option<tonic::Code> {
let body: serde_json::Value = serde_json::from_str(body).ok()?;
let value = body.get("grpcCodeValue")?.as_i64()?;
Some(tonic::Code::from(i32::try_from(value).ok()?))
}
fn http_error_info(body: &str) -> Option<ErrorInfo> {
let body: serde_json::Value = serde_json::from_str(body).ok()?;
let reason = body.get("code")?.as_str()?;
if reason.is_empty() || reason == "NA" {
return None;
}
let metadata = body
.get("context")
.and_then(serde_json::Value::as_object)
.map(|context| {
context
.iter()
.map(|(key, value)| {
let value = match value.as_str() {
Some(text) => text.to_string(),
None => value.to_string(),
};
(key.clone(), value)
})
.collect()
})
.unwrap_or_default();
Some(ErrorInfo {
reason: reason.to_string(),
domain: String::new(),
metadata,
})
}
fn http_correlation_id(body: &str) -> Option<String> {
let body: serde_json::Value = serde_json::from_str(body).ok()?;
["correlationId", "traceId"]
.iter()
.find_map(|key| Some(body.get(key)?.as_str()?.to_string()))
}
fn parse_spelled_duration(text: &str) -> Option<std::time::Duration> {
let mut words = text.split_whitespace();
let amount: f64 = words.next()?.parse().ok()?;
let unit = words.next()?;
if words.next().is_some() {
return None;
}
let seconds = match unit.strip_suffix('s').unwrap_or(unit) {
"day" => amount * 86_400.0,
"hour" => amount * 3_600.0,
"minute" => amount * 60.0,
"second" => amount,
"millisecond" => amount / 1e3,
"microsecond" => amount / 1e6,
"nanosecond" => amount / 1e9,
_ => return None,
};
std::time::Duration::try_from_secs_f64(seconds).ok()
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum ErrorCategory {
TransientServerFailure,
ContentionOnSharedResources,
DeadlineExceededRequestStateUnknown,
SystemInternalAssumptionViolated,
SecurityAlert,
AuthInterceptorInvalidAuthenticationCredentials,
InsufficientPermission,
InvalidIndependentOfSystemState,
InvalidGivenCurrentSystemStateOther,
InvalidGivenCurrentSystemStateResourceExists,
InvalidGivenCurrentSystemStateResourceMissing,
InvalidGivenCurrentSystemStateSeekAfterEnd,
InternalUnsupportedOperation,
}
impl TryFrom<i32> for ErrorCategory {
type Error = i32;
fn try_from(id: i32) -> std::result::Result<Self, i32> {
Self::from_i32(id).ok_or(id)
}
}
impl From<ErrorCategory> for i32 {
fn from(category: ErrorCategory) -> i32 {
category.as_i32()
}
}
impl ErrorCategory {
#[must_use]
pub const fn from_i32(id: i32) -> Option<Self> {
Some(match id {
1 => Self::TransientServerFailure,
2 => Self::ContentionOnSharedResources,
3 => Self::DeadlineExceededRequestStateUnknown,
4 => Self::SystemInternalAssumptionViolated,
5 => Self::SecurityAlert,
6 => Self::AuthInterceptorInvalidAuthenticationCredentials,
7 => Self::InsufficientPermission,
8 => Self::InvalidIndependentOfSystemState,
9 => Self::InvalidGivenCurrentSystemStateOther,
10 => Self::InvalidGivenCurrentSystemStateResourceExists,
11 => Self::InvalidGivenCurrentSystemStateResourceMissing,
12 => Self::InvalidGivenCurrentSystemStateSeekAfterEnd,
14 => Self::InternalUnsupportedOperation,
_ => return None,
})
}
#[must_use]
pub const fn as_i32(self) -> i32 {
match self {
Self::TransientServerFailure => 1,
Self::ContentionOnSharedResources => 2,
Self::DeadlineExceededRequestStateUnknown => 3,
Self::SystemInternalAssumptionViolated => 4,
Self::SecurityAlert => 5,
Self::AuthInterceptorInvalidAuthenticationCredentials => 6,
Self::InsufficientPermission => 7,
Self::InvalidIndependentOfSystemState => 8,
Self::InvalidGivenCurrentSystemStateOther => 9,
Self::InvalidGivenCurrentSystemStateResourceExists => 10,
Self::InvalidGivenCurrentSystemStateResourceMissing => 11,
Self::InvalidGivenCurrentSystemStateSeekAfterEnd => 12,
Self::InternalUnsupportedOperation => 14,
}
}
#[must_use]
pub const fn is_retriable(self) -> bool {
matches!(
self,
Self::TransientServerFailure
| Self::ContentionOnSharedResources
| Self::DeadlineExceededRequestStateUnknown
| Self::InvalidGivenCurrentSystemStateSeekAfterEnd
)
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct ErrorInfo {
pub reason: String,
pub domain: String,
pub metadata: std::collections::HashMap<String, String>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct ResourceInfo {
pub resource_type: String,
pub resource_name: String,
pub owner: String,
pub description: String,
}
impl From<tonic::Status> for Error {
fn from(status: tonic::Status) -> Self {
Error::Status(Box::new(status))
}
}
impl From<tonic::transport::Error> for Error {
fn from(err: tonic::transport::Error) -> Self {
Error::Transport(Box::new(err))
}
}
impl From<serde_json::Error> for Error {
fn from(err: serde_json::Error) -> Self {
Error::Json(Box::new(err))
}
}
pub type Result<T, E = Error> = std::result::Result<T, E>;
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use super::*;
#[test]
fn error_info_is_extracted_from_a_status_and_absent_otherwise() {
use tonic_types::{ErrorDetails, StatusExt as _};
let mut metadata = std::collections::HashMap::new();
metadata.insert("resource".to_string(), "contract-1".to_string());
let details = ErrorDetails::with_error_info("DUPLICATE_COMMAND", "canton", metadata);
let status = tonic::Status::with_error_details(tonic::Code::AlreadyExists, "dup", details);
let info = Error::from(status)
.error_info()
.expect("error info present");
assert_eq!(info.reason, "DUPLICATE_COMMAND");
assert_eq!(info.domain, "canton");
assert_eq!(
info.metadata.get("resource").map(String::as_str),
Some("contract-1")
);
assert!(
Error::from(tonic::Status::not_found("x"))
.error_info()
.is_none()
);
assert!(Error::Timeout.error_info().is_none());
}
fn canton_status(
code: tonic::Code,
category: i32,
delay: Option<std::time::Duration>,
) -> tonic::Status {
use tonic_types::{ErrorDetails, StatusExt as _};
let mut metadata = std::collections::HashMap::new();
metadata.insert("category".to_string(), category.to_string());
let mut details = ErrorDetails::with_error_info("SOME_ERROR_CODE", "participant", metadata);
details.set_request_info("corr-1234", "");
if let Some(delay) = delay {
details.set_retry_info(Some(delay));
}
tonic::Status::with_error_details(code, "boom", details)
}
#[test]
fn the_canton_category_decides_retryability_over_the_grpc_code() {
use std::time::Duration;
let err = Error::from(canton_status(tonic::Code::Aborted, 10, None));
assert_eq!(
err.category(),
Some(ErrorCategory::InvalidGivenCurrentSystemStateResourceExists)
);
assert!(!err.is_retriable());
let err = Error::from(canton_status(
tonic::Code::OutOfRange,
12,
Some(Duration::from_secs(1)),
));
assert_eq!(
err.category(),
Some(ErrorCategory::InvalidGivenCurrentSystemStateSeekAfterEnd)
);
assert!(err.is_retriable());
assert_eq!(err.retry_delay(), Some(Duration::from_secs(1)));
}
#[test]
fn correlation_id_and_retry_delay_are_extracted() {
use std::time::Duration;
let err = Error::from(canton_status(
tonic::Code::Unavailable,
1,
Some(Duration::from_millis(250)),
));
assert_eq!(err.category(), Some(ErrorCategory::TransientServerFailure));
assert_eq!(err.correlation_id().as_deref(), Some("corr-1234"));
assert_eq!(err.retry_delay(), Some(Duration::from_millis(250)));
let plain = Error::from(tonic::Status::unavailable("x"));
assert_eq!(plain.category(), None);
assert_eq!(plain.correlation_id(), None);
assert_eq!(plain.retry_delay(), None);
assert_eq!(Error::Timeout.category(), None);
}
#[test]
fn statuses_without_a_category_fall_back_to_code_classification() {
use tonic_types::{ErrorDetails, StatusExt as _};
assert!(Error::from(tonic::Status::unavailable("x")).is_retriable());
assert!(!Error::from(tonic::Status::not_found("x")).is_retriable());
let err = Error::from(canton_status(tonic::Code::Unavailable, 99, None));
assert_eq!(err.category(), None);
assert!(err.is_retriable());
let mut details = ErrorDetails::new();
details.set_retry_info(Some(std::time::Duration::from_secs(2)));
let status =
tonic::Status::with_error_details(tonic::Code::FailedPrecondition, "wait", details);
assert!(Error::from(status).is_retriable());
}
#[test]
fn json_api_error_bodies_classify_by_category() {
let body = r#"{
"code": "OFFSET_AFTER_LEDGER_END",
"cause": "Begin offset (999999999) is after ledger end (23577)",
"correlationId": null,
"traceId": "36a33702b2fa7908a7349be166ccfa38",
"context": {"participant": "'app-provider'", "category": "12"},
"resources": [],
"errorCategory": 12,
"grpcCodeValue": 11,
"retryInfo": "1 second",
"definiteAnswer": null
}"#;
let err = Error::Http {
status: 400,
body: body.to_string(),
};
assert_eq!(
err.category(),
Some(ErrorCategory::InvalidGivenCurrentSystemStateSeekAfterEnd)
);
assert!(err.is_retriable());
assert_eq!(err.retry_delay(), Some(std::time::Duration::from_secs(1)));
assert_eq!(
err.correlation_id().as_deref(),
Some("36a33702b2fa7908a7349be166ccfa38")
);
assert_eq!(err.code(), Some(tonic::Code::OutOfRange));
let info = err.error_info().expect("the body names the error");
assert_eq!(info.reason, "OFFSET_AFTER_LEDGER_END");
assert_eq!(
info.metadata.get("category").map(String::as_str),
Some("12")
);
assert_eq!(
info.metadata.get("participant").map(String::as_str),
Some("'app-provider'")
);
let err = Error::Http {
status: 503,
body: r#"{"errorCategory": 8}"#.to_string(),
};
assert_eq!(
err.category(),
Some(ErrorCategory::InvalidIndependentOfSystemState)
);
assert!(!err.is_retriable());
}
#[test]
fn non_json_http_bodies_fall_back_to_status_code_classification() {
let retriable = Error::Http {
status: 503,
body: "<html>Service Unavailable</html>".to_string(),
};
assert!(retriable.is_retriable());
assert_eq!(retriable.category(), None);
assert_eq!(retriable.retry_delay(), None);
let terminal = Error::Http {
status: 404,
body: String::new(),
};
assert!(!terminal.is_retriable());
assert_eq!(terminal.correlation_id(), None);
}
#[test]
fn spelled_durations_parse_and_garbage_is_refused() {
use std::time::Duration;
for (text, expected) in [
("1 second", Duration::from_secs(1)),
("5 seconds", Duration::from_secs(5)),
("250 milliseconds", Duration::from_millis(250)),
("2 minutes", Duration::from_secs(120)),
("1 hour", Duration::from_secs(3600)),
("0.5 seconds", Duration::from_millis(500)),
] {
assert_eq!(parse_spelled_duration(text), Some(expected), "{text}");
}
for bad in [
"",
"soon",
"1",
"1 fortnight",
"-1 second",
"1 second ago",
"1e300 seconds",
"1e300 days",
"NaN seconds",
"inf seconds",
] {
assert_eq!(parse_spelled_duration(bad), None, "{bad}");
}
}
#[test]
fn category_ids_round_trip_and_follow_the_docs_retryability() {
for id in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14] {
let category = ErrorCategory::from_i32(id).expect("known id");
assert_eq!(category.as_i32(), id);
assert_eq!(category.is_retriable(), matches!(id, 1 | 2 | 3 | 12));
}
assert_eq!(ErrorCategory::from_i32(0), None);
assert_eq!(ErrorCategory::from_i32(13), None);
assert_eq!(ErrorCategory::from_i32(15), None);
}
#[test]
fn a_redacted_status_yields_nothing_except_the_correlation_id() {
use tonic_types::{ErrorDetails, StatusExt as _};
let mut details = ErrorDetails::new();
details.set_request_info("93199811c5b2090c51cf45fe8c88060c", "");
let status = tonic::Status::with_error_details(
tonic::Code::Unauthenticated,
"An error occurred. Please contact the operator and inquire about the request \
93199811c5b2090c51cf45fe8c88060c",
details,
);
let err = Error::from(status);
assert_eq!(err.category(), None, "a redacted status has no category");
assert_eq!(err.error_info(), None);
assert!(err.resource_info().is_empty());
assert_eq!(err.retry_delay(), None);
assert_eq!(
err.correlation_id().as_deref(),
Some("93199811c5b2090c51cf45fe8c88060c"),
"the correlation id is the only actionable thing left"
);
assert!(!err.is_retriable());
let body = r#"{"code":"NA","cause":"An error occurred. Please contact the operator",
"errorCategory":-1,"retryInfo":null,"resources":[],
"correlationId":"41f217564e4e76f6cbc853a94a82fa80",
"traceId":"41f217564e4e76f6cbc853a94a82fa80"}"#;
let err = Error::Http {
status: 401,
body: body.to_string(),
};
assert_eq!(err.category(), None, "-1 is not a category");
assert!(err.resource_info().is_empty());
assert_eq!(err.retry_delay(), None);
assert_eq!(err.error_info(), None, "\"NA\" is not an error id");
assert_eq!(
err.correlation_id().as_deref(),
Some("41f217564e4e76f6cbc853a94a82fa80")
);
assert!(!err.is_retriable(), "401 is not transient");
}
#[test]
fn resource_info_names_what_the_error_is_about() {
use tonic_types::{ErrorDetails, StatusExt as _};
let mut details = ErrorDetails::new();
details.set_resource_info("ErrorResource(CONTRACT_ID)", "00abc", "alice", "not found");
let status = tonic::Status::with_error_details(tonic::Code::NotFound, "gone", details);
let found = Error::from(status).resource_info();
assert_eq!(found.len(), 1);
assert_eq!(found[0].resource_type, "ErrorResource(CONTRACT_ID)");
assert_eq!(found[0].resource_name, "00abc");
assert_eq!(found[0].owner, "alice");
assert!(
Error::from(tonic::Status::not_found("x"))
.resource_info()
.is_empty()
);
assert!(Error::Timeout.resource_info().is_empty());
}
#[test]
fn resource_info_reads_the_json_apis_resources_array() {
let body = r#"{"code":"CONTRACT_NOT_FOUND","cause":"…","errorCategory":11,
"resources":[["ErrorResource(CONTRACT_ID)","00ababab"]],"retryInfo":null}"#;
let err = Error::Http {
status: 404,
body: body.to_string(),
};
let found = err.resource_info();
assert_eq!(found.len(), 1);
assert_eq!(found[0].resource_type, "ErrorResource(CONTRACT_ID)");
assert_eq!(found[0].resource_name, "00ababab");
assert!(found[0].owner.is_empty());
let many = Error::Http {
status: 409,
body: r#"{"resources":[["A","1"],["B","2"]]}"#.to_string(),
};
assert_eq!(many.resource_info().len(), 2);
let ragged = Error::Http {
status: 500,
body: r#"{"resources":[["A"],["B","2"],42,null]}"#.to_string(),
};
assert_eq!(ragged.resource_info().len(), 1);
for body in [r#"{"resources":[]}"#, "{}", "not json at all", ""] {
let err = Error::Http {
status: 500,
body: body.to_string(),
};
assert!(err.resource_info().is_empty(), "{body}");
}
}
#[test]
fn transient_conditions_are_retriable() {
assert!(Error::Timeout.is_retriable());
assert!(Error::Connection("reset".to_string()).is_retriable());
assert!(Error::from(tonic::Status::unavailable("x")).is_retriable());
assert!(Error::from(tonic::Status::deadline_exceeded("x")).is_retriable());
assert!(Error::from(tonic::Status::resource_exhausted("x")).is_retriable());
assert!(Error::from(tonic::Status::aborted("x")).is_retriable());
}
#[test]
fn transient_http_codes_are_retriable_but_client_codes_are_not() {
for status in [
408, 429, 500, 501, 502, 503, 504, 507, 509, 511, 520, 527, 599,
] {
assert!(
Error::Http {
status,
body: String::new()
}
.is_retriable(),
"http {status} should be retriable"
);
}
for status in [400, 401, 403, 404, 409, 413, 422] {
assert!(
!Error::Http {
status,
body: String::new()
}
.is_retriable(),
"http {status} should not be retriable"
);
}
}
#[test]
fn definite_failures_are_not_retriable() {
assert!(!Error::from(tonic::Status::not_found("x")).is_retriable());
assert!(!Error::from(tonic::Status::already_exists("dup")).is_retriable());
assert!(!Error::from(tonic::Status::invalid_argument("x")).is_retriable());
assert!(!Error::InvalidRequest("x".to_string()).is_retriable());
assert!(!Error::Auth("x".to_string()).is_retriable());
assert!(
!Error::CommandRejected {
code: "GrpcStatus".to_string(),
message: "boom".to_string()
}
.is_retriable()
);
assert!(!Error::UnexpectedResponse("x".to_string()).is_retriable());
}
#[test]
fn code_is_exposed_only_for_status_errors() {
assert_eq!(
Error::from(tonic::Status::not_found("x")).code(),
Some(tonic::Code::NotFound)
);
assert_eq!(Error::Timeout.code(), None);
assert_eq!(Error::Connection("x".to_string()).code(), None);
assert_eq!(
Error::Http {
status: 503,
body: String::new()
}
.code(),
None
);
}
#[test]
fn display_messages_are_lowercase_and_informative() {
assert_eq!(Error::Timeout.to_string(), "operation timed out");
assert_eq!(
Error::InvalidRequest("bad uri".to_string()).to_string(),
"invalid request: bad uri"
);
assert_eq!(
Error::Auth("token expired".to_string()).to_string(),
"authentication failed: token expired"
);
assert_eq!(
Error::Http {
status: 503,
body: "down".to_string()
}
.to_string(),
"http 503: down"
);
assert_eq!(
Error::CommandRejected {
code: "INVALID_ARGUMENT".to_string(),
message: "nope".to_string()
}
.to_string(),
"command rejected (INVALID_ARGUMENT): nope"
);
}
}