ironoxide/

internal.rs

//! Common types, traits, and functions needed across user/group/document apis
//! If it can be defined in API specific file, it should go there to keep this file's
//! size to a minimum.

use crate::Result;
use base64::engine::Engine;
use base64::prelude::BASE64_STANDARD;
use futures::Future;
use group_api::GroupId;
use itertools::{Either, Itertools};
use lazy_static::lazy_static;
use log::error;
use papaya::HashMap;
use protobuf::{self, Error as ProtobufError};
use quick_error::quick_error;
use recrypt::api::{
    CryptoOps, Ed25519, Ed25519Signature, Hashable, KeyGenOps, Plaintext,
    PrivateKey as RecryptPrivateKey, PublicKey as RecryptPublicKey, RandomBytes, Recrypt,
    RecryptErr, Sha256, SigningKeypair as RecryptSigningKeypair,
};
use regex::Regex;
use reqwest::Method;
use rest::{Authorization, SignatureUrlString};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::{
    convert::{TryFrom, TryInto},
    fmt::{Error, Formatter},
    result::Result as StdResult,
    sync::{Mutex, MutexGuard},
};
use time::OffsetDateTime;
use user_api::UserId;

pub mod document_api;
pub mod group_api;
mod rest;
pub mod user_api;
pub use rest::IronCoreRequest;

const DEVICE_SIGNATURE_LENGTH: usize = 64;

lazy_static! {
    pub static ref URL_STRING: String = match std::env::var("IRONCORE_ENV") {
        Ok(url) => match url.to_lowercase().as_ref() {
            "stage" => "https://api-staging.ironcorelabs.com/api/1/",
            "prod" => "https://api.ironcorelabs.com/api/1/",
            url_choice => url_choice,
        }
        .to_string(),
        _ => "https://api.ironcorelabs.com/api/1/".to_string(),
    };
}

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum RequestErrorCode {
    UserVerify,
    UserCreate,
    UserUpdate,
    UserUpdateStatus,
    UserDeviceAdd,
    UserDeviceDelete,
    UserDeviceList,
    UserKeyList,
    UserKeyUpdate,
    UserGetCurrent,
    GroupCreate,
    GroupDelete,
    GroupList,
    GroupGet,
    GroupAddMember,
    GroupUpdate,
    GroupMemberRemove,
    GroupAdminRemove,
    GroupKeyUpdate,
    DocumentList,
    DocumentGet,
    DocumentCreate,
    DocumentUpdate,
    DocumentGrantAccess,
    DocumentRevokeAccess,
    EdekTransform,
    PolicyGet,
}

/// Public SDK operations
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum SdkOperation {
    InitializeSdk,
    InitializeSdkCheckRotation,
    RotateAll,
    DocumentList,
    DocumentGetMetadata,
    DocumentEncrypt,
    DocumentUpdateBytes,
    DocumentDecrypt,
    DocumentUpdateName,
    DocumentGrantAccess,
    DocumentRevokeAccess,
    DocumentEncryptUnmanaged,
    DocumentDecryptUnmanaged,
    DocumentFileEncrypt,
    DocumentFileDecrypt,
    DocumentFileEncryptUnmanaged,
    DocumentFileDecryptUnmanaged,
    UserCreate,
    UserListDevices,
    GenerateNewDevice,
    UserDeleteDevice,
    UserVerify,
    UserUpdateStatus,
    UserDisableSelf,
    UserGetPublicKey,
    UserRotatePrivateKey,
    UserChangePassword,
    GroupList,
    GroupCreate,
    GroupGetMetadata,
    GroupDelete,
    GroupUpdateName,
    GroupAddMembers,
    GroupRemoveMembers,
    GroupAddAdmins,
    GroupRemoveAdmins,
    GroupRotatePrivateKey,
}

impl std::fmt::Display for SdkOperation {
    fn fmt(&self, f: &mut Formatter<'_>) -> StdResult<(), Error> {
        write!(f, "'{self:?}'")
    }
}

quick_error! {
    /// Errors generated by IronOxide SDK operations
    #[derive(Debug)]
    #[non_exhaustive]
    pub enum IronOxideErr {
        ValidationError(field_name: String, err: String) {
            display("'{}' failed validation with the error '{}'", field_name, err)
        }
        DocumentHeaderParseFailure(message: String) {
            display("{}", message)
        }
        WrongSizeError(actual_size: Option<usize>, expected_size: Option<usize>) {
        }
        KeyGenerationError {
            display("Key generation failed")
        }
        AesError(err: aws_lc_rs::error::Unspecified) {
            source(err)
        }
        AesEncryptedDocSizeError{
            display("Provided document is not long enough to be an encrypted document.")
        }
        InvalidRecryptEncryptedValue(msg: String) {
            display("Got an unexpected Recrypt EncryptedValue: '{}'", msg)
        }
        RecryptError(msg: String) {
            display("Recrypt operation failed with error '{}'", msg)
        }
        UserDoesNotExist(msg: String) {
            display("Operation failed with error '{}'", msg)
        }
        UserOrGroupDoesNotExist(user_or_group: document_api::UserOrGroup) {
            display("User or group {} does not exist.", user_or_group)
        }
        InitializeError(cause: String) {
            display("SDK initialization failed. Underlying cause '{}'", cause)
        }
        RequestError { message: String, code: RequestErrorCode, http_status: Option<u16> } {
            display("Request failed with HTTP status code '{:?}' message '{}' and code '{:?}'", http_status, message, code)
        }
        ///This is used if the response from the server was an error. In that case we know that the format of the errors will be `ServerError`.
        RequestServerErrors {errors: Vec<rest::ServerError>, code: RequestErrorCode, http_status: Option<u16> } {
            display("Request failed with HTTP status code '{:?}' errors list is '{:?}' and code '{:?}'", http_status, errors, code)
        }
        MissingTransformBlocks {
            display("Expected at least one TransformBlock in transformed value but received none.")
        }
        ///The operation failed because the accessing user was not a group admin, but must be for the operation to work.
        NotGroupAdmin(id: GroupId) {
            display("You are not an administrator of group '{}'", id.id())
        }
        /// No policy exists for the segment
        PolicyDoesNotExist {
            display("No policy is defined. Please visit https://admin.ironcorelabs.com/policy to set a policy")
        }
        /// Protobuf encode/decode error
        ProtobufSerdeError(err: ProtobufError) {
            source(err)
        }
        /// Protobuf decode succeeded, but the result is not valid
        ProtobufValidationError(msg: String) {
            display("Protobuf validation failed with '{}'", msg)
        }
        UnmanagedDecryptionError(edek_doc_id: String, edek_segment_id: i32,
                                 edoc_doc_id: String, edoc_segment_id: i32) {
            display("Edeks and EncryptedDocument do not match. \
            Edeks are for DocumentId({}) and SegmentId({}) and\
            Encrypted Document is DocumentId({}) and SegmentId({})",
            edek_doc_id, edek_segment_id, edoc_doc_id, edoc_segment_id)
        }
        UserPrivateKeyRotationError(msg: String) {
            display("User private key rotation failed with '{}'", msg)
        }
        GroupPrivateKeyRotationError(msg: String) {
            display("Group private key rotation failed with '{}'", msg)
        }
        OperationTimedOut{operation: SdkOperation, duration: std::time::Duration} {
            display("Operation {} timed out after {}ms", operation, duration.as_millis())
        }
        JoinError(msg: String) {
            display("{}", msg)
        }
        CacheSerdeError(error: postcard::Error) {
            source(error)
        }
        AesGcmDecryptError {
            display("AES-GCM decryption failed: authentication tag verification failed")
        }
        FileIoError { path: Option<String>, operation: String, message: String } {
            display("File I/O error {}during {}: {}", path.as_ref().map(|s| format!("for '{s}' ")).unwrap_or("".to_string()), operation, message)
        }
    }
}

/// A way to turn IronSdkErr into Strings for the Java binding
impl From<IronOxideErr> for String {
    fn from(err: IronOxideErr) -> Self {
        err.to_string()
    }
}

impl From<RecryptErr> for IronOxideErr {
    fn from(recrypt_err: RecryptErr) -> Self {
        match recrypt_err {
            RecryptErr::InputWrongSize(_, expected_size) => {
                IronOxideErr::WrongSizeError(None, Some(expected_size))
            }
            RecryptErr::InvalidPublicKey(_) => IronOxideErr::KeyGenerationError,
            //Fallback for all other error types that Recrypt can have that we don't have specific mappings for
            other_recrypt_err => IronOxideErr::RecryptError(other_recrypt_err.to_string()),
        }
    }
}

impl From<ProtobufError> for IronOxideErr {
    fn from(e: ProtobufError) -> Self {
        IronOxideErr::ProtobufSerdeError(e)
    }
}

impl From<recrypt::nonemptyvec::NonEmptyVecError> for IronOxideErr {
    fn from(_: recrypt::nonemptyvec::NonEmptyVecError) -> Self {
        IronOxideErr::MissingTransformBlocks
    }
}

impl From<tokio::task::JoinError> for IronOxideErr {
    fn from(e: tokio::task::JoinError) -> Self {
        IronOxideErr::JoinError(e.to_string())
    }
}

const NAME_AND_ID_MAX_LEN: usize = 100;

/// Validate that the provided id is valid for our user/document/group IDs. Validates that the
/// ID has a length and that it matches our restricted set of characters. Also takes the readable
/// type of ID for usage within any resulting error messages.
pub fn validate_id(id: &str, id_type: &str) -> Result<String> {
    let id_regex = Regex::new("^[a-zA-Z0-9_.$#|@/:;=+'-]+$").expect("regex is valid");
    let trimmed_id = id.trim();
    if trimmed_id.is_empty() || trimmed_id.len() > NAME_AND_ID_MAX_LEN {
        Err(IronOxideErr::ValidationError(
            id_type.to_string(),
            format!("'{trimmed_id}' must have length between 1 and 100"),
        ))
    } else if !id_regex.is_match(trimmed_id) {
        Err(IronOxideErr::ValidationError(
            id_type.to_string(),
            format!("'{trimmed_id}' contains invalid characters"),
        ))
    } else {
        Ok(trimmed_id.to_string())
    }
}

/// Validate that the provided document/group name is valid. Ensures that the length of
/// the name is between 1-100 characters. Also takes the readable type of the name for
/// usage within any resulting error messages.
pub fn validate_name(name: &str, name_type: &str) -> Result<String> {
    let trimmed_name = name.trim();
    if trimmed_name.trim().is_empty() || trimmed_name.len() > NAME_AND_ID_MAX_LEN {
        Err(IronOxideErr::ValidationError(
            name_type.to_string(),
            format!("'{trimmed_name}' must have length between 1 and 100"),
        ))
    } else {
        Ok(trimmed_name.trim().to_string())
    }
}

pub mod auth_v2 {
    use time::OffsetDateTime;

    use super::*;

    /// API Auth version 2.
    /// Fully constructing a valid auth v2 header is a two step process.
    /// Step 1 is done on construction via `new`
    /// Step 2 is done via `finish_with` as a request is being sent out and the bytes of the body are available.
    pub struct AuthV2Builder<'a> {
        pub(in crate::internal::auth_v2) req_auth: &'a RequestAuth,
        pub(in crate::internal::auth_v2) timestamp: OffsetDateTime,
    }

    impl AuthV2Builder<'_> {
        pub fn new(req_auth: &RequestAuth, timestamp: OffsetDateTime) -> AuthV2Builder<'_> {
            AuthV2Builder {
                req_auth,
                timestamp,
            }
        }

        /// Always returns Authorization::Version2
        /// # Arguments
        /// `sig_url`       URL path to be signed over
        /// `method`        Method of request (POST, GET, PUT, etc)
        /// `body_bytes`    Reference to the bytes of the body (or none)
        ///
        /// # Returns
        /// Authorization::Version2 that contains all the information necessary to make an
        /// IronCore authenticated request to the webservice.
        pub fn finish_with<'a>(
            &'a self,
            sig_url: SignatureUrlString,
            method: Method,
            body_bytes: Option<&'a [u8]>,
        ) -> Authorization<'a> {
            self.req_auth
                .create_signature_v2(self.timestamp, sig_url, method, body_bytes)
        }
    }
}

///Structure that contains all the info needed to make a signed API request from a device.
#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RequestAuth {
    ///The user's given id, which uniquely identifies them inside the segment.
    account_id: UserId,
    ///The segment_id for the above user.
    segment_id: usize,
    ///The signing key which was generated for the device. “expanded private key” (both pub/priv)
    signing_private_key: DeviceSigningKeyPair,
    #[serde(skip_serializing, skip_deserializing)]
    pub(crate) request: IronCoreRequest,
}
impl RequestAuth {
    pub fn create_signature_v2<'a>(
        &'a self,
        current_time: OffsetDateTime,
        sig_url: SignatureUrlString,
        method: Method,
        body: Option<&'a [u8]>,
    ) -> Authorization<'a> {
        Authorization::create_signatures_v2(
            current_time,
            self.segment_id,
            &self.account_id,
            method,
            sig_url,
            body,
            &self.signing_private_key,
        )
    }

    pub fn account_id(&self) -> &UserId {
        &self.account_id
    }

    pub fn segment_id(&self) -> usize {
        self.segment_id
    }

    pub fn signing_private_key(&self) -> &DeviceSigningKeyPair {
        &self.signing_private_key
    }
}

/// Signing and encryption key pairs and metadata for a device.
///
/// Required to initialize the SDK with a set of device keys (see [ironoxide::initialize](../fn.initialize.html)).
///
/// Can be generated by calling [generate_new_device](../user/trait.UserOps.html#tymethod.generate_new_device) and
/// passing the result to `DeviceContext::from`.
#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DeviceContext {
    #[serde(flatten)]
    auth: RequestAuth,
    /// The private key which was generated for a particular device for the user. Not the user's master private key.
    device_private_key: PrivateKey,
}
impl DeviceContext {
    /// Constructs a `DeviceContext` from its components.
    ///
    /// To instead generate a new `DeviceContext` for the user, call [generate_new_device](../user/trait.UserOps.html#tymethod.generate_new_device)
    /// and pass the result to `DeviceContext::from`.
    pub fn new(
        account_id: UserId,
        segment_id: usize,
        device_private_key: PrivateKey,
        signing_private_key: DeviceSigningKeyPair,
    ) -> DeviceContext {
        DeviceContext {
            auth: RequestAuth {
                account_id,
                segment_id,
                signing_private_key,
                request: IronCoreRequest::default(),
            },
            device_private_key,
        }
    }

    pub(crate) fn auth(&self) -> &RequestAuth {
        &self.auth
    }
    /// ID of the device's owner
    pub fn account_id(&self) -> &UserId {
        &self.auth.account_id
    }
    /// ID of the segment
    pub fn segment_id(&self) -> usize {
        self.auth.segment_id
    }
    /// Private signing key of the device
    pub fn signing_private_key(&self) -> &DeviceSigningKeyPair {
        &self.auth.signing_private_key
    }
    /// Private encryption key of the device
    pub fn device_private_key(&self) -> &PrivateKey {
        &self.device_private_key
    }
}

type UserPublicKeyCache = HashMap<UserId, PublicKey>;
type GroupPublicKeyCache = HashMap<GroupId, PublicKey>;
/// A cache recording the (id, public key) pairs that have been seen by the API. There are
/// separate lists for users and groups.
// Public keys don't go bad, so we don't need expiration, but we may want a default limit on size in the future.
#[derive(Serialize, Deserialize, Debug)]
pub(crate) struct PublicKeyCache {
    /// The public key of the user that created this public key cache.
    /// Public key caches are only valid in the context of the one user, and we need their public key to initialize.
    creator_public_key: PublicKey,
    #[serde(
        serialize_with = "serialize_papaya_map",
        deserialize_with = "deserialize_papaya_map"
    )]
    user_keys: UserPublicKeyCache,
    #[serde(
        serialize_with = "serialize_papaya_map",
        deserialize_with = "deserialize_papaya_map"
    )]
    group_keys: GroupPublicKeyCache,
}

/// `papaya` can't provide `size_hint` (due to being concurrent, no reliable upper bound) to indicate to serde's
/// `serialize_seq` how long it is during serialization. `postcard` (our data format) requires a length on
/// `serialize_seq` or it errors.
/// This manual serde gets around concurrency issues by snapshotting a Vec of entries at call time and serializing that.
fn serialize_papaya_map<S, K, V>(map: &HashMap<K, V>, serializer: S) -> StdResult<S::Ok, S::Error>
where
    S: Serializer,
    K: Serialize + core::hash::Hash + Eq,
    V: Serialize,
{
    serializer.collect_seq(map.pin().iter().collect::<Vec<_>>())
}
fn deserialize_papaya_map<'de, D, K, V>(deserializer: D) -> StdResult<HashMap<K, V>, D::Error>
where
    D: Deserializer<'de>,
    K: Deserialize<'de> + core::hash::Hash + Eq,
    V: Deserialize<'de>,
{
    let entries: Vec<(K, V)> = Vec::deserialize(deserializer)?;
    let map = HashMap::new();
    {
        // scoped to drop the pin's guard before return the map
        let pinned_map = map.pin();
        for (k, v) in entries {
            pinned_map.insert(k, v);
        }
    }
    Ok(map)
}

impl PublicKeyCache {
    pub(crate) fn new(current_user_public_key: &PublicKey) -> Self {
        Self {
            creator_public_key: current_user_public_key.clone(),
            user_keys: Default::default(),
            group_keys: Default::default(),
        }
    }
    /// Serialize the cache to bytes that can be persisted and reloaded
    /// when the SDK is initialized
    pub(crate) fn serialize(&self) -> Result<Vec<u8>> {
        postcard::to_stdvec(&self).map_err(IronOxideErr::CacheSerdeError)
    }
    pub(crate) fn deserialize(serialized_cache: &[u8]) -> Result<Self> {
        postcard::from_bytes(serialized_cache).map_err(IronOxideErr::CacheSerdeError)
    }
    /// The public key of the user that created this cache, used for offline SDK initialization
    pub(crate) fn creator_public_key(&self) -> &PublicKey {
        &self.creator_public_key
    }
    pub(crate) fn user_keys(&self) -> &HashMap<UserId, PublicKey> {
        &self.user_keys
    }
    pub(crate) fn group_keys(&self) -> &HashMap<GroupId, PublicKey> {
        &self.group_keys
    }
    pub(crate) fn deserialize_signed_public_key_cache(
        device: &DeviceContext,
        signed_cache_bytes: &[u8],
    ) -> Result<PublicKeyCache> {
        if signed_cache_bytes.len() < DEVICE_SIGNATURE_LENGTH {
            return Err(IronOxideErr::WrongSizeError(
                Some(signed_cache_bytes.len()),
                Some(DEVICE_SIGNATURE_LENGTH),
            ));
        }
        let (signature, cache) = signed_cache_bytes.split_at(DEVICE_SIGNATURE_LENGTH);
        if device.signing_private_key().verify(&cache, signature)? {
            Self::deserialize(cache)
        } else {
            Err(IronOxideErr::ValidationError(
                "signed public key cache".to_string(),
                "The signed public key cache failed signature verification.".to_string(),
            ))
        }
    }
    pub(crate) fn serialize_signed_public_key_cache(
        &self,
        device: &DeviceContext,
    ) -> Result<Vec<u8>> {
        self.serialize().map(|cache_bytes| {
            // sign the bytes, then create a result of signature + cache
            let signature = device.signing_private_key().sign(&cache_bytes);
            let mut signed_cache = Vec::new();
            signed_cache.extend_from_slice(&signature);
            signed_cache.extend(cache_bytes);
            signed_cache
        })
    }
}

// helper function for getting keys from an API via a cache first, and updating the cache after.
pub(crate) async fn get_keys_with_cache<Id, F, Op>(
    ids: &[Id],
    cache: &HashMap<Id, PublicKey>,
    fetch: F,
) -> Result<(Vec<Id>, Vec<WithKey<Id>>)>
where
    Id: Clone + Eq + core::hash::Hash,
    F: FnOnce(Vec<Id>) -> Op,
    Op: Future<Output = Result<std::collections::HashMap<Id, PublicKey>>>,
{
    // if there aren't any ids in the list, just return with empty results
    if ids.is_empty() {
        return Ok((vec![], vec![]));
    }

    let (cached, uncached): (Vec<_>, Vec<_>) = ids.iter().cloned().partition_map(|id| match cache
        .pin()
        .get(&id)
    {
        Some(pub_key) => Either::Left(WithKey::new(id, pub_key.clone())),
        None => Either::Right(id),
    });

    // everything requested was cached, we can return without making an API call
    if uncached.is_empty() {
        return Ok((vec![], cached));
    }

    // call the API for remaining missing ids
    let ids_with_keys = fetch(uncached.clone()).await?;
    let (not_found, found): (Vec<_>, Vec<_>) =
        uncached
            .into_iter()
            .partition_map(|id| match ids_with_keys.get(&id).cloned() {
                Some(pub_key) => {
                    // cache the newly returned API values
                    cache.pin().insert(id.clone(), pub_key.clone());
                    Either::Right(WithKey::new(id, pub_key))
                }
                None => Either::Left(id),
            });

    Ok((not_found, [cached, found].concat()))
}

/// Newtype wrapper around Recrypt TransformKey type
#[derive(Clone, PartialEq, Eq, Debug)]
pub struct TransformKey(recrypt::api::TransformKey);
impl From<recrypt::api::TransformKey> for TransformKey {
    fn from(tk: recrypt::api::TransformKey) -> Self {
        TransformKey(tk)
    }
}
impl Hashable for TransformKey {
    fn to_bytes(&self) -> Vec<u8> {
        self.0.to_bytes()
    }
}

/// Newtype wrapper around Recrypt SchnorrSignature type
#[derive(Clone, PartialEq, Eq, Debug)]
pub struct SchnorrSignature(recrypt::api::SchnorrSignature);
impl From<recrypt::api::SchnorrSignature> for SchnorrSignature {
    fn from(s: recrypt::api::SchnorrSignature) -> Self {
        SchnorrSignature(s)
    }
}
impl From<SchnorrSignature> for Vec<u8> {
    fn from(sig: SchnorrSignature) -> Self {
        sig.0.bytes().to_vec()
    }
}

/// Asymmetric public encryption key.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct PublicKey(RecryptPublicKey);
impl PublicKey {
    fn to_bytes_x_y(&self) -> (Vec<u8>, Vec<u8>) {
        let (x, y) = self.0.bytes_x_y();
        (x.to_vec(), y.to_vec())
    }
    pub fn new_from_slice(bytes: (&[u8], &[u8])) -> Result<Self> {
        let re_pub = RecryptPublicKey::new_from_slice(bytes)?;
        Ok(PublicKey(re_pub))
    }
    /// Bytes of the public key
    pub fn as_bytes(&self) -> Vec<u8> {
        let (mut x, mut y) = self.to_bytes_x_y();
        x.append(&mut y);
        x
    }
}
impl From<RecryptPublicKey> for PublicKey {
    fn from(recrypt_pub: RecryptPublicKey) -> Self {
        PublicKey(recrypt_pub)
    }
}
impl From<PublicKey> for RecryptPublicKey {
    fn from(public_key: PublicKey) -> Self {
        public_key.0
    }
}
impl From<&PublicKey> for RecryptPublicKey {
    fn from(public_key: &PublicKey) -> Self {
        public_key.0
    }
}
impl From<PublicKey> for crate::proto::transform::PublicKey {
    fn from(pubk: PublicKey) -> Self {
        crate::proto::transform::PublicKey {
            x: pubk.to_bytes_x_y().0.into(),
            y: pubk.to_bytes_x_y().1.into(),
            ..Default::default()
        }
    }
}
impl TryFrom<&[u8]> for PublicKey {
    type Error = IronOxideErr;
    fn try_from(key_bytes: &[u8]) -> Result<PublicKey> {
        if key_bytes.len() == RecryptPublicKey::ENCODED_SIZE_BYTES {
            PublicKey::new_from_slice(key_bytes.split_at(RecryptPublicKey::ENCODED_SIZE_BYTES / 2))
        } else {
            Err(IronOxideErr::WrongSizeError(
                Some(RecryptPublicKey::ENCODED_SIZE_BYTES),
                Some(key_bytes.len()),
            ))
        }
    }
}
impl Serialize for PublicKey {
    fn serialize<S: Serializer>(&self, serializer: S) -> StdResult<S::Ok, S::Error> {
        self.as_bytes().serialize(serializer)
    }
}
impl<'de> Deserialize<'de> for PublicKey {
    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> StdResult<Self, D::Error> {
        let bytes: Vec<u8> = Deserialize::deserialize(deserializer)?;
        if bytes.len() != 64 {
            return Err(serde::de::Error::invalid_length(bytes.len(), &"64 bytes"));
        }
        let (x, y) = bytes.split_at(32);
        PublicKey::new_from_slice((x, y)).map_err(serde::de::Error::custom)
    }
}

/// Asymmetric private encryption key.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct PrivateKey(RecryptPrivateKey);
impl PrivateKey {
    const BYTES_SIZE: usize = RecryptPrivateKey::ENCODED_SIZE_BYTES;
    /// Bytes of the private key
    pub fn as_bytes(&self) -> &[u8; PrivateKey::BYTES_SIZE] {
        self.0.bytes()
    }
    fn recrypt_key(&self) -> &RecryptPrivateKey {
        &self.0
    }
    /// Augment this private key with another, producing a new PrivateKey
    fn augment<F: FnOnce(String) -> IronOxideErr>(
        &self,
        augmenting_key: &AugmentationFactor,
        error_fn: F,
    ) -> Result<PrivateKey> {
        let zero: RecryptPrivateKey = RecryptPrivateKey::new([0u8; 32]);
        if RecryptPrivateKey::from(augmenting_key.clone()) == zero {
            Err(error_fn("Augmenting key cannot be zero".into()))
        } else if RecryptPrivateKey::from(augmenting_key.clone()) == self.0 {
            Err(error_fn(
                "PrivateKey augmentation failed with a zero value".into(),
            ))
        } else {
            // this subtraction needs to be the additive inverse of what the service is doing
            let augmented_key = self.0.augment_minus(&augmenting_key.clone().into());
            Ok(augmented_key.into())
        }
    }
    /// A convenience function to pass a user rotation error to `augment()`
    fn augment_user(&self, augmenting_key: &AugmentationFactor) -> Result<PrivateKey> {
        self.augment(augmenting_key, IronOxideErr::UserPrivateKeyRotationError)
    }
    /// A convenience function to pass a user rotation error to `augment()`
    fn augment_group(&self, augmenting_key: &AugmentationFactor) -> Result<PrivateKey> {
        self.augment(augmenting_key, IronOxideErr::GroupPrivateKeyRotationError)
    }
}
impl From<RecryptPrivateKey> for PrivateKey {
    fn from(recrypt_priv: RecryptPrivateKey) -> Self {
        PrivateKey(recrypt_priv)
    }
}
impl From<PrivateKey> for RecryptPrivateKey {
    fn from(priv_key: PrivateKey) -> Self {
        priv_key.0
    }
}
impl From<[u8; 32]> for PrivateKey {
    fn from(bytes: [u8; 32]) -> Self {
        PrivateKey(RecryptPrivateKey::new(bytes))
    }
}
impl TryFrom<&[u8]> for PrivateKey {
    type Error = IronOxideErr;
    fn try_from(key_bytes: &[u8]) -> Result<PrivateKey> {
        RecryptPrivateKey::new_from_slice(key_bytes)
            .map(PrivateKey)
            .map_err(|e| e.into())
    }
}
impl Serialize for PrivateKey {
    fn serialize<S>(&self, serializer: S) -> StdResult<S::Ok, S::Error>
    where
        S: Serializer,
    {
        serializer.serialize_str(&BASE64_STANDARD.encode(self.0.bytes()))
    }
}
impl<'de> Deserialize<'de> for PrivateKey {
    fn deserialize<D>(deserializer: D) -> StdResult<Self, D::Error>
    where
        D: Deserializer<'de>,
    {
        use serde::de::Error;
        let s = String::deserialize(deserializer)?;
        let keys_bytes = BASE64_STANDARD
            .decode(s)
            .map_err(|e| Error::custom(e.to_string()))?;
        PrivateKey::try_from(&keys_bytes[..]).map_err(|e| Error::custom(e.to_string()))
    }
}

/// Private key used to augment another PrivateKey
#[derive(Clone, Debug)]
pub(crate) struct AugmentationFactor(PrivateKey);
impl AugmentationFactor {
    /// Use recrypt to generate a new AugmentationFactor
    pub fn generate_new<R: KeyGenOps>(recrypt: &R) -> AugmentationFactor {
        AugmentationFactor(recrypt.random_private_key().into())
    }

    pub fn as_bytes(&self) -> &[u8; 32] {
        self.0.as_bytes()
    }
}
impl From<AugmentationFactor> for RecryptPrivateKey {
    fn from(aug: AugmentationFactor) -> Self {
        (aug.0).0
    }
}

/// Key pair used to sign all requests to the IronCore API endpoints.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct DeviceSigningKeyPair(RecryptSigningKeypair);
impl DeviceSigningKeyPair {
    pub fn sign(&self, payload: &[u8]) -> [u8; DEVICE_SIGNATURE_LENGTH] {
        self.0.sign(&payload).into()
    }
    /// Bytes of the signing key pair
    pub fn as_bytes(&self) -> &[u8; 64] {
        self.0.bytes()
    }
    pub fn public_key(&self) -> [u8; 32] {
        self.0.public_key().into()
    }
    pub fn verify<A: Hashable>(&self, message: &A, signature: &[u8]) -> Result<bool> {
        let ed25519_signature = Ed25519Signature::new_from_slice(signature)?;
        Ok(self.0.public_key().verify(message, &ed25519_signature))
    }
}
impl From<&DeviceSigningKeyPair> for RecryptSigningKeypair {
    fn from(dsk: &DeviceSigningKeyPair) -> RecryptSigningKeypair {
        dsk.0.clone()
    }
}
impl From<RecryptSigningKeypair> for DeviceSigningKeyPair {
    fn from(rsk: RecryptSigningKeypair) -> DeviceSigningKeyPair {
        DeviceSigningKeyPair(rsk)
    }
}
impl TryFrom<&[u8]> for DeviceSigningKeyPair {
    type Error = IronOxideErr;
    fn try_from(signing_key_bytes: &[u8]) -> Result<DeviceSigningKeyPair> {
        RecryptSigningKeypair::from_byte_slice(signing_key_bytes)
            .map(DeviceSigningKeyPair)
            .map_err(|e| {
                IronOxideErr::ValidationError("DeviceSigningKeyPair".to_string(), e.to_string())
            })
    }
}
impl Serialize for DeviceSigningKeyPair {
    fn serialize<S>(&self, serializer: S) -> StdResult<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let base64 = BASE64_STANDARD.encode(self.0.bytes());
        serializer.serialize_str(&base64)
    }
}
impl<'de> Deserialize<'de> for DeviceSigningKeyPair {
    fn deserialize<D>(deserializer: D) -> StdResult<Self, D::Error>
    where
        D: Deserializer<'de>,
    {
        use serde::de::Error;
        let s = String::deserialize(deserializer)?;
        let keys_bytes = BASE64_STANDARD
            .decode(s)
            .map_err(|e| Error::custom(e.to_string()))?;
        DeviceSigningKeyPair::try_from(&keys_bytes[..]).map_err(|e| Error::custom(e.to_string()))
    }
}

/// Newtype wrapper around a string which represents the users master private key escrow password
#[derive(Debug, PartialEq, Eq)]
pub struct Password(String);
impl TryFrom<&str> for Password {
    type Error = IronOxideErr;
    fn try_from(maybe_password: &str) -> Result<Self> {
        if !maybe_password.trim().is_empty() {
            Ok(Password(maybe_password.to_string()))
        } else {
            Err(IronOxideErr::ValidationError(
                "maybe_password".to_string(),
                "length must be > 0".to_string(),
            ))
        }
    }
}

#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct WithKey<T> {
    pub(crate) id: T,
    pub(crate) public_key: PublicKey,
}
impl<T> WithKey<T> {
    pub fn new(id: T, public_key: PublicKey) -> WithKey<T> {
        WithKey { id, public_key }
    }
}

/// Acquire mutex in a blocking fashion. If the Mutex is or becomes poisoned, write out an error
/// message and panic.
///
/// The lock is released when the returned MutexGuard falls out of scope.
///
/// # Usage:
/// single statement (mut)
/// `let result = take_lock(&t).deref_mut().call_method_on_t();`
///
/// multi-statement (mut)
/// ```ignore
/// let t = T {};
/// let result = {
///     let g = &mut *take_lock(&t);
///     g.call_method_on_t()
/// }; // lock released here
/// ```
///
pub(crate) fn take_lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
    m.lock().unwrap_or_else(|e| {
        let error = format!("Error when acquiring lock: {e}");
        error!("{}", error);
        panic!("{}", error);
    })
}

/// Attempts to augment an existing private key with a newly generated augmentation factor.
/// There is a very small chance that an augmentation factor could not be compatible with
/// the given PrivateKey, so we retry once internally before giving the caller an error.
fn augment_private_key_with_retry<R: KeyGenOps>(
    recrypt: &R,
    priv_key: &PrivateKey,
) -> Result<(PrivateKey, AugmentationFactor)> {
    let aug_private_key = || {
        let aug_factor = AugmentationFactor::generate_new(recrypt);
        priv_key.augment_user(&aug_factor).map(|p| (p, aug_factor))
    };
    // retry generation of augmentation factor one time. If this fails twice there's something wrong.
    aug_private_key().or_else(|_| aug_private_key())
}

/// Subtracts a generated private key from the provided PrivateKey, returning
/// the result and the plaintext associated with the generated key.
/// There is a very small chance that the generated private key could not be compatible with
/// the given PrivateKey, so we retry once internally before giving the caller an error.
fn gen_plaintext_and_aug_with_retry<R: CryptoOps>(
    recrypt: &R,
    priv_key: &PrivateKey,
) -> Result<(Plaintext, AugmentationFactor)> {
    let aug_private_key = || -> Result<(Plaintext, AugmentationFactor)> {
        let new_plaintext = recrypt.gen_plaintext();
        let new_group_private_key = recrypt.derive_private_key(&new_plaintext);
        let new_key_aug = AugmentationFactor(new_group_private_key.into());
        let aug_factor = priv_key.augment_group(&new_key_aug)?;
        Ok((new_plaintext, AugmentationFactor(aug_factor)))
    };
    // retry generation of private key one time. If this fails twice there's something wrong.
    aug_private_key().or_else(|_| aug_private_key())
}

/// Runs a future with a timeout or just runs the future, depending on if a timeout is specified.
///
/// If a timeout limit is reached, the result will be an IronOxideErr::OperationTimedOut.
/// If no timeout is specified, or if the operation finishes before the timeout, the
/// result is the result of the sdk operation.
pub async fn add_optional_timeout<F: Future>(
    f: F,
    timeout: Option<std::time::Duration>,
    op: SdkOperation,
) -> Result<F::Output> {
    use futures::future::TryFutureExt;
    let result = match timeout {
        Some(d) => {
            tokio::time::timeout(d, f)
                .map_err(|_| IronOxideErr::OperationTimedOut {
                    operation: op,
                    duration: d,
                })
                .await?
        }

        // no timeout, just run the Future and return
        None => f.await,
    };

    Ok(result)
}

#[cfg(test)]
pub(crate) mod tests {
    use super::*;
    use double::*;
    use galvanic_assert::{matchers::*, *};
    use recrypt::api::Ed25519Ops;
    use std::fmt::Debug;
    use tokio::time::Duration;
    use vec1::vec1;

    /// String contains matcher to assert that the provided substring exists in the provided value
    pub fn contains(expected: &str) -> Box<dyn Matcher<'_, String> + '_> {
        Box::new(move |actual: &String| {
            let builder = MatchResultBuilder::for_("contains");
            if actual.contains(expected) {
                builder.matched()
            } else {
                let expected_string: String = expected.to_string();
                builder.failed_comparison(actual, &expected_string)
            }
        })
    }

    /// Length matcher to assert that the provided iterable value has the expected size
    pub fn length<'a, I, T>(expected: &'a usize) -> Box<dyn Matcher<'a, I> + 'a>
    where
        T: 'a,
        &'a I: Debug + Sized + IntoIterator<Item = &'a T> + 'a,
    {
        Box::new(move |actual: &'a I| {
            let actual_list: Vec<_> = actual.into_iter().collect();
            let builder = MatchResultBuilder::for_("contains");
            if &actual_list.len() == expected {
                builder.matched()
            } else {
                builder.failed_because(&format!(
                    "Expected '{:?}' to have length of {} but found length of {}",
                    actual,
                    expected,
                    actual_list.len()
                ))
            }
        })
    }

    #[test]
    fn serde_devicecontext_roundtrip() -> Result<()> {
        let context = create_test_device_context();
        let json = serde_json::to_string(&context).unwrap();
        let expect_json = r#"{"accountId":"account_id","segmentId":22,"signingPrivateKey":"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQGKiOPddAnxlf1S2y08ul1yymcJvx2UEhvzdIgBtA9vXA==","devicePrivateKey":"bzb0Rlg0u7gx9wDuk1ppRI77OH/0ferXleenJ3Ag6Jg="}"#;

        assert_eq!(json, expect_json);

        let de: DeviceContext = serde_json::from_str(&json).unwrap();

        assert_eq!(context.account_id(), de.account_id());
        assert_eq!(
            context.auth.signing_private_key.as_bytes().to_vec(),
            de.auth.signing_private_key.as_bytes().to_vec()
        );
        assert_eq!(
            context.device_private_key.as_bytes().to_vec(),
            de.device_private_key.as_bytes().to_vec()
        );
        Ok(())
    }

    #[test]
    fn validate_id_success() {
        let valid_id = "abcABC012_.$#|@/:;=+'-";
        let id = validate_id(valid_id, "id_type");
        assert_that!(&id, is_variant!(Ok));
        assert_that!(&id.unwrap(), eq(valid_id.to_string()))
    }

    #[test]
    fn valid_id_whitespace() {
        let valid_id = " abc212     ";
        let id = validate_id(valid_id, "id_type");
        assert_that!(&id, is_variant!(Ok));
        assert_that!(&id.unwrap(), eq("abc212".to_string()))
    }

    #[test]
    fn validate_id_failure() {
        let invalid_id = "with spaces";
        let id_type = "id_type";
        let id = validate_id(invalid_id, id_type);
        assert_that!(&id, is_variant!(Err));
        let validation_error = id.unwrap_err();
        assert_that!(
            &validation_error,
            is_variant!(IronOxideErr::ValidationError)
        );
        assert_that!(&format!("{}", validation_error), contains(id_type));
        assert_that!(&format!("{}", validation_error), contains(invalid_id));
    }

    #[test]
    fn validate_id_all_whitespace() {
        let invalid_id = "     ";
        let id_type = "id_type";
        let id = validate_id(invalid_id, id_type);
        assert_that!(&id, is_variant!(Err));
        let validation_error = id.unwrap_err();
        assert_that!(
            &validation_error,
            is_variant!(IronOxideErr::ValidationError)
        );
        assert_that!(&format!("{}", validation_error), contains(id_type));
    }

    #[test]
    fn validate_name_success() {
        let valid_name = "name with any char _.$#|@/:;=+'-";
        let id = validate_name(valid_name, "name_type");
        assert_that!(&id, is_variant!(Ok));
        assert_that!(&id.unwrap(), eq(valid_name.to_string()))
    }

    #[test]
    fn validate_name_surrounding_whitespace() {
        let valid_name = "   a good name    ";
        let id = validate_name(valid_name, "name_type");
        assert_that!(&id, is_variant!(Ok));
        assert_that!(&id.unwrap(), eq("a good name".to_string()))
    }

    #[test]
    fn validate_name_failure() {
        let name_type = "name_type";
        let invalid_name = "too many chars 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
        let name = validate_name(invalid_name, name_type);
        assert_that!(&name, is_variant!(Err));
        let validation_error = name.unwrap_err();
        assert_that!(
            &validation_error,
            is_variant!(IronOxideErr::ValidationError)
        );
        assert_that!(&format!("{}", validation_error), contains(invalid_name));
        assert_that!(&format!("{}", validation_error), contains(name_type));
    }

    #[test]
    fn validate_name_all_whitespace() {
        let invalid_name = "        ";
        let name_type = "name_type";

        let name = validate_name(invalid_name, name_type);
        assert_that!(&name, is_variant!(Err));
        let validation_error = name.unwrap_err();
        assert_that!(
            &validation_error,
            is_variant!(IronOxideErr::ValidationError)
        );
        assert_that!(&format!("{}", validation_error), contains(name_type));
    }

    #[test]
    fn passphrase_validation() {
        let result = Password::try_from("");
        assert!(result.is_err())
    }

    #[test]
    fn encode_proto_public_key() -> Result<()> {
        let recr = recrypt::api::Recrypt::new();
        let (_, re_pubk) = recr.generate_key_pair()?;
        let pubk: PublicKey = re_pubk.into();

        let proto_pubk: crate::proto::transform::PublicKey = pubk.clone().into();
        assert_eq!(
            (&pubk.to_bytes_x_y().0, &pubk.to_bytes_x_y().1),
            (&proto_pubk.x.to_vec(), &proto_pubk.y.to_vec())
        );
        Ok(())
    }
    #[test]
    fn public_key_postcard_roundtrip() {
        let recr = Recrypt::new();
        let (_, re_pubk) = recr.generate_key_pair().unwrap();
        let pubk: PublicKey = re_pubk.into();

        let bytes = postcard::to_stdvec(&pubk).unwrap();
        let deserialized: PublicKey = postcard::from_bytes(&bytes).unwrap();
        assert_eq!(pubk, deserialized);
    }

    #[test]
    fn public_key_deserialize_wrong_length_fails() {
        let bytes = postcard::to_stdvec(&vec![0u8; 30]).unwrap(); // too short for a public key
        let result: StdResult<PublicKey, _> = postcard::from_bytes(&bytes);
        assert!(result.is_err());
    }

    #[test]
    fn public_key_try_from_slice() -> Result<()> {
        let recr = recrypt::api::Recrypt::new();
        let (_, re_pubk) = recr.generate_key_pair()?;
        let pubk: PublicKey = re_pubk.into();
        let pubk2: PublicKey = pubk.as_bytes().as_slice().try_into()?;
        assert_eq!(pubk, pubk2);
        Ok(())
    }

    #[test]
    fn public_key_try_from_slice_invalid() {
        let bytes = [1u8; 8];
        let maybe_public_key: Result<PublicKey> = bytes[..].try_into();
        assert!(maybe_public_key.is_err())
    }

    pub fn gen_priv_key() -> PrivateKey {
        let recr = recrypt::api::Recrypt::new();
        let (re_privk, _) = recr.generate_key_pair().unwrap();
        re_privk.into()
    }

    #[test]
    fn private_key_augment_with_self_is_none() {
        let privk = gen_priv_key();

        let result = privk.augment_user(&AugmentationFactor(privk.clone()));
        assert_that!(&result, is_variant!(Err));
        assert_that!(
            &result.unwrap_err(),
            is_variant!(IronOxideErr::UserPrivateKeyRotationError)
        )
    }

    #[test]
    fn private_key_augmentation_is_augment_minus() {
        let p1 = gen_priv_key();
        let p2 = gen_priv_key();

        let p3 = p1.0.augment_minus(&p2.0);

        let aug_p = p1.augment_user(&AugmentationFactor(p2)).unwrap();
        assert_eq!(aug_p.0, p3)
    }

    #[test]
    fn private_key_augmentation_aug_key_of_zero_is_err() {
        let priv_key_orig = gen_priv_key();
        let zero_aug_factor = AugmentationFactor(PrivateKey(RecryptPrivateKey::new([0u8; 32])));
        let new_priv_key = priv_key_orig.augment_user(&zero_aug_factor);
        assert_that!(&new_priv_key, is_variant!(Err));
        assert_that!(
            &new_priv_key.unwrap_err(),
            is_variant!(IronOxideErr::UserPrivateKeyRotationError)
        )
    }

    mock_trait!(
        MockKeyGenOps,
        random_private_key() -> recrypt::api::PrivateKey
    );
    impl KeyGenOps for MockKeyGenOps {
        fn compute_public_key(
            &self,
            _private_key: &RecryptPrivateKey,
        ) -> StdResult<RecryptPublicKey, RecryptErr> {
            unimplemented!()
        }

        mock_method!(random_private_key(&self) -> RecryptPrivateKey);

        fn generate_key_pair(
            &self,
        ) -> StdResult<(RecryptPrivateKey, RecryptPublicKey), RecryptErr> {
            unimplemented!()
        }

        fn generate_transform_key(
            &self,
            _from_private_key: &RecryptPrivateKey,
            _to_public_key: &RecryptPublicKey,
            _signing_keypair: &recrypt::api::SigningKeypair,
        ) -> StdResult<recrypt::api::TransformKey, RecryptErr> {
            unimplemented!()
        }
    }
    mock_trait!(MockCryptoOps,
        gen_plaintext() -> recrypt::api::Plaintext
    );
    impl CryptoOps for MockCryptoOps {
        fn derive_symmetric_key(
            &self,
            _: &recrypt::api::Plaintext,
        ) -> recrypt::api::DerivedSymmetricKey {
            unimplemented!()
        }
        mock_method!(gen_plaintext(&self) -> recrypt::api::Plaintext);
        fn transform(
            &self,
            _: recrypt::api::EncryptedValue,
            _: recrypt::api::TransformKey,
            _: &recrypt::api::SigningKeypair,
        ) -> StdResult<recrypt::api::EncryptedValue, RecryptErr> {
            unimplemented!()
        }
        fn decrypt(
            &self,
            _: recrypt::api::EncryptedValue,
            _: &recrypt::api::PrivateKey,
        ) -> StdResult<recrypt::api::Plaintext, RecryptErr> {
            unimplemented!()
        }
        fn encrypt(
            &self,
            _: &recrypt::api::Plaintext,
            _: &recrypt::api::PublicKey,
            _: &recrypt::api::SigningKeypair,
        ) -> StdResult<recrypt::api::EncryptedValue, RecryptErr> {
            unimplemented!()
        }
        fn derive_private_key(&self, pt: &recrypt::api::Plaintext) -> recrypt::api::PrivateKey {
            let recrypt = recrypt::api::Recrypt::new();
            recrypt.derive_private_key(pt)
        }
    }
    #[test]
    fn augment_private_key_with_retry_retries_once() {
        let recrypt_mock = MockKeyGenOps::default();
        let good_re_private_key = RecryptPrivateKey::new([42u8; 32]); // good private key
        recrypt_mock.random_private_key.return_values(vec![
            RecryptPrivateKey::new([0u8; 32]), // bad private key, 0s
            good_re_private_key.clone(),       // good private key. Used for aug factor
        ]);

        let curr_priv_key = PrivateKey::from([100u8; 32]);
        let expected_priv_key_bytes: [u8; 32] = [
            58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58,
            58, 58, 58, 58, 58, 58, 58, 58, 58, 58,
        ];

        let result = augment_private_key_with_retry(&recrypt_mock, &curr_priv_key).unwrap();
        assert_eq!(
            (result.0).0,
            RecryptPrivateKey::new(expected_priv_key_bytes)
        );
        assert_eq!(((result.1).0).0, good_re_private_key)
    }
    #[test]
    fn augment_private_key_with_retry_retries_only_once() {
        let recrypt_mock = MockKeyGenOps::default();
        recrypt_mock.random_private_key.return_values(vec![
            RecryptPrivateKey::new([0u8; 32]),   // bad private key, 0s
            RecryptPrivateKey::new([100u8; 32]), // bad private key, matches current
            RecryptPrivateKey::new([42u8; 32]),  // good private key, never returned
        ]);

        let curr_priv_key = PrivateKey::from([100u8; 32]);

        let result = augment_private_key_with_retry(&recrypt_mock, &curr_priv_key);
        assert_that!(
            &result.unwrap_err(),
            is_variant!(IronOxideErr::UserPrivateKeyRotationError)
        );
    }
    #[test]
    fn gen_plaintext_and_diff_with_retry_retries_once() {
        let recrypt_mock = MockCryptoOps::default();
        // creating a real recrypt to make a valid plaintext
        let recrypt = recrypt::api::Recrypt::new();
        let bad_plaintext = recrypt.gen_plaintext();
        let bad_private_key = recrypt.derive_private_key(&bad_plaintext);
        let good_plaintext = recrypt.gen_plaintext();
        recrypt_mock
            .gen_plaintext
            .return_values(vec![bad_plaintext, good_plaintext.clone()]);

        // since this will generate bad_plaintext, which bad_private_key is derived from,
        // the augmentation will result in zero, causing the function to retry.
        let result =
            gen_plaintext_and_aug_with_retry(&recrypt_mock, &bad_private_key.into()).unwrap();
        assert_eq!(result.0, good_plaintext);
    }

    #[test]
    fn gen_plaintext_and_diff_with_retry_retries_only_once() {
        let recrypt_mock = MockCryptoOps::default();
        // creating a real recrypt to make a valid plaintext
        let recrypt = recrypt::api::Recrypt::new();
        let bad_plaintext = recrypt.gen_plaintext();
        let bad_private_key = recrypt.derive_private_key(&bad_plaintext);
        let good_plaintext = recrypt.gen_plaintext();
        // Ideally this would also check that it retries/fails when the generated private key is zero,
        // but I don't know the plaintext to return to force that to happen.
        // Mocking `derive_private_key()` doesn't appear to be possible without Eq and Hash on Plaintext.
        recrypt_mock.gen_plaintext.return_values(vec![
            bad_plaintext.clone(),
            bad_plaintext,
            good_plaintext,
        ]);

        // since this will generate bad_plaintext, which bad_private_key is derived from,
        // the augmentation will result in zero, causing the function to retry.
        let result = gen_plaintext_and_aug_with_retry(&recrypt_mock, &bad_private_key.into());
        assert_that!(
            &result.unwrap_err(),
            is_variant!(IronOxideErr::GroupPrivateKeyRotationError)
        );
    }

    #[test]
    fn init_and_rotation_user_and_groups() -> Result<()> {
        use crate::{
            InitAndRotationCheck, IronOxide, check_groups_and_collect_rotation,
            internal::{
                group_api::tests::create_group_meta_result, user_api::tests::create_user_result,
            },
        };
        let recrypt = recrypt::api::Recrypt::new();
        let (_, pub_key) = recrypt.generate_key_pair()?;
        let time = OffsetDateTime::now_utc();
        let create_gmr = |id: GroupId, needs_rotation: Option<bool>| {
            create_group_meta_result(
                id,
                None,
                pub_key.into(),
                true,
                true,
                time,
                time,
                needs_rotation,
            )
        };
        let de_json = r#"{"deviceId":314,"accountId":"account_id","segmentId":22,"signingPrivateKey":"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQGKiOPddAnxlf1S2y08ul1yymcJvx2UEhvzdIgBtA9vXA==","devicePrivateKey":"bzb0Rlg0u7gx9wDuk1ppRI77OH/0ferXleenJ3Ag6Jg="}"#;
        let de: DeviceContext = serde_json::from_str(de_json).unwrap();
        let user_id = UserId::try_from("account_id")?;
        let user = create_user_result(user_id.clone(), 22, pub_key.into(), true);
        let io = IronOxide::create(&user, &de, &Default::default());

        let good_group_id = GroupId::try_from("group")?;
        let gmr_vec = vec![
            create_gmr(good_group_id.clone(), Some(true)),
            create_gmr(GroupId::try_from("notthisone")?, Some(false)),
            create_gmr(GroupId::try_from("northisone")?, None),
        ];
        let init = check_groups_and_collect_rotation(&gmr_vec, true, user_id.clone(), io);
        let rotation = match init {
            InitAndRotationCheck::NoRotationNeeded(_) => panic!("user and group need rotation"),
            InitAndRotationCheck::RotationNeeded(_, rotation) => rotation,
        };
        assert_eq!(
            rotation.group_rotation_needed(),
            Some(&vec1![good_group_id])
        );
        assert_eq!(rotation.user_rotation_needed(), Some(&user_id));
        Ok(())
    }

    #[tokio::test]
    async fn run_maybe_timed_sdk_op_no_timeout() -> Result<()> {
        async fn get_42() -> u8 {
            tokio::time::sleep(Duration::from_millis(100)).await;
            42
        }
        let forty_two = get_42();
        let result =
            add_optional_timeout(forty_two, None, SdkOperation::DocumentRevokeAccess).await?;
        assert_eq!(result, 42);

        let forty_two = get_42();
        let result = add_optional_timeout(
            forty_two,
            Some(Duration::from_secs(1)),
            SdkOperation::DocumentRevokeAccess,
        )
        .await?;
        assert_eq!(result, 42);

        async fn get_err() -> Result<()> {
            tokio::time::sleep(Duration::from_millis(100)).await;
            Err(IronOxideErr::MissingTransformBlocks)
        }

        let err_f = get_err();
        let result = add_optional_timeout(err_f, None, SdkOperation::DocumentRevokeAccess).await?;
        assert!(result.is_err());
        assert_that!(
            &result.unwrap_err(),
            is_variant!(IronOxideErr::MissingTransformBlocks)
        );

        let err_f = get_err();
        let result = add_optional_timeout(
            err_f,
            Some(Duration::from_secs(1)),
            SdkOperation::DocumentRevokeAccess,
        )
        .await?;
        assert!(result.is_err());
        assert_that!(
            &result.unwrap_err(),
            is_variant!(IronOxideErr::MissingTransformBlocks)
        );

        Ok(())
    }

    #[tokio::test]
    async fn run_maybe_timed_sdk_op_with_timeout() -> Result<()> {
        async fn get_42() -> u8 {
            // allow other futures to run, like the timer
            // without this the future will run to completion, regardless of the timer
            tokio::time::sleep(Duration::from_millis(100)).await;
            42
        }

        let forty_two = get_42();
        let result = add_optional_timeout(
            forty_two,
            Some(Duration::from_nanos(1)),
            SdkOperation::DocumentRevokeAccess,
        )
        .await;
        assert!(result.is_err());
        assert_that!(
            &result.unwrap_err(),
            is_variant!(IronOxideErr::OperationTimedOut)
        );

        async fn get_err() -> Result<u8> {
            tokio::time::sleep(Duration::from_millis(100)).await;
            Err(IronOxideErr::MissingTransformBlocks)
        }

        let err_f = get_err();
        let result = add_optional_timeout(
            err_f,
            Some(Duration::from_millis(1)),
            SdkOperation::DocumentRevokeAccess,
        )
        .await;
        assert!(result.is_err());
        assert_that!(
            &result.unwrap_err(),
            is_variant!(IronOxideErr::OperationTimedOut)
        );
        Ok(())
    }
    #[test]
    fn signing_key_sign_then_verify() {
        let recrypt = Recrypt::new();
        let signing_keypair = recrypt.generate_ed25519_key_pair();
        let device_keypair = DeviceSigningKeyPair::from(signing_keypair);

        let message = b"test payload";
        let signature = device_keypair.sign(message);
        assert!(
            device_keypair
                .verify(&message.as_slice(), &signature)
                .unwrap()
        );
    }
    #[test]
    fn signing_key_verify_wrong_message_fails() {
        let recrypt = Recrypt::new();
        let signing_keypair = recrypt.generate_ed25519_key_pair();
        let device_keypair = DeviceSigningKeyPair::from(signing_keypair);

        let signature = device_keypair.sign(b"original");
        assert!(
            !device_keypair
                .verify(&b"tampered".as_slice(), &signature)
                .unwrap()
        );
    }
    #[test]
    fn signing_key_verify_bad_signature_length_fails() {
        let recrypt = Recrypt::new();
        let signing_keypair = recrypt.generate_ed25519_key_pair();
        let device_keypair = DeviceSigningKeyPair::from(signing_keypair);

        let result = device_keypair.verify(&b"message".as_slice(), &[0u8; 32]);
        assert!(result.is_err());
    }
    #[test]
    fn empty_public_key_cache_roundtrip() -> Result<()> {
        let recr = recrypt::api::Recrypt::new();
        let (_, re_pubk) = recr.generate_key_pair()?;
        let cache = PublicKeyCache::new(&re_pubk.into());
        let bytes = cache.serialize().unwrap();
        let deserialized = PublicKeyCache::deserialize(&bytes).unwrap();
        assert_eq!(deserialized.user_keys().len(), 0);
        assert_eq!(deserialized.group_keys().len(), 0);
        Ok(())
    }
    #[test]
    fn populated_public_key_cache_roundtrip() {
        let recrypt = Recrypt::new();
        let (_, pub1) = recrypt.generate_key_pair().unwrap();
        let (_, pub2) = recrypt.generate_key_pair().unwrap();

        let cache = PublicKeyCache::new(&pub1.into());
        cache
            .user_keys()
            .pin()
            .insert(UserId::unsafe_from_string("user1".into()), pub1.into());
        cache
            .group_keys()
            .pin()
            .insert(GroupId::unsafe_from_string("group1".into()), pub2.into());

        let bytes = cache.serialize().unwrap();
        let deserialized = PublicKeyCache::deserialize(&bytes).unwrap();

        let user_id = UserId::unsafe_from_string("user1".into());
        let group_id = GroupId::unsafe_from_string("group1".into());
        assert!(deserialized.user_keys().pin().get(&user_id).is_some());
        assert!(deserialized.group_keys().pin().get(&group_id).is_some());
    }
    #[test]
    fn public_key_cache_deserialize_garbage_bytes_fails() {
        let result = PublicKeyCache::deserialize(b"deadbeef");
        assert!(result.is_err());
    }
    pub(crate) fn create_test_device_context() -> DeviceContext {
        let priv_key: recrypt::api::PrivateKey = recrypt::api::PrivateKey::new_from_slice(
            BASE64_STANDARD
                .decode("bzb0Rlg0u7gx9wDuk1ppRI77OH/0ferXleenJ3Ag6Jg=")
                .unwrap()
                .as_slice(),
        )
        .unwrap();
        let dev_keys = recrypt::api::SigningKeypair::from_byte_slice(&[
            1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
            1, 1, 1, 138, 136, 227, 221, 116, 9, 241, 149, 253, 82, 219, 45, 60, 186, 93, 114, 202,
            103, 9, 191, 29, 148, 18, 27, 243, 116, 136, 1, 180, 15, 111, 92,
        ])
        .unwrap();
        DeviceContext::new(
            "account_id".try_into().unwrap(),
            22,
            priv_key.into(),
            DeviceSigningKeyPair::from(dev_keys),
        )
    }

    pub(crate) fn create_test_sdk() -> Result<crate::IronOxide> {
        use crate::{IronOxide, internal::user_api::tests::create_user_result};
        let recrypt = Recrypt::new();
        let device = create_test_device_context();
        let user_id = UserId::try_from("account_id")?;
        let (_, pubk) = recrypt.generate_key_pair()?;
        let user = create_user_result(user_id.clone(), 22, pubk.into(), true);
        let io = IronOxide::create(&user, &device, &Default::default());
        Ok(io)
    }

    mod signed {
        use super::*;
        #[test]
        fn signed_cache_roundtrip() -> Result<()> {
            let io = create_test_sdk()?;
            let recrypt = Recrypt::new();
            let device = super::create_test_device_context();
            let (_, pubk) = recrypt.generate_key_pair()?;

            let user_id = UserId::unsafe_from_string("user1".into());
            io.public_key_cache
                .user_keys()
                .pin()
                .insert(user_id.clone(), pubk.into());

            let signed = io.export_public_key_cache()?;
            let result = PublicKeyCache::deserialize_signed_public_key_cache(&device, &signed)?;
            // Cache contains 2 entries: the current user (account_id) and the inserted user (user1)
            assert!(result.user_keys().len() == 2);
            let user_keys = result.user_keys().pin();
            let deser_pubk = user_keys
                .get(&user_id)
                .expect("expected inserted user to exist");
            assert_eq!(pubk, deser_pubk.0);
            Ok(())
        }
        #[test]
        fn signed_cache_tampered_payload_fails() -> Result<()> {
            let io = create_test_sdk()?;
            let mut signed = io.export_public_key_cache()?;
            // flip a byte in the cache portion
            let last = signed.len() - 1;
            signed[last] ^= 0xFF;

            let result = PublicKeyCache::deserialize_signed_public_key_cache(io.device(), &signed);
            assert!(result.is_err());
            Ok(())
        }
        #[test]
        fn signed_cache_tampered_signature_fails() -> Result<()> {
            let io = create_test_sdk()?;
            let mut signed = io.export_public_key_cache()?;
            // flip a byte in the signature portion
            signed[0] ^= 0xFF;

            let result = PublicKeyCache::deserialize_signed_public_key_cache(io.device(), &signed);
            assert!(result.is_err());
            Ok(())
        }
        #[test]
        fn signed_cache_wrong_device_fails() -> Result<()> {
            let io = create_test_sdk()?;
            let signed = io.export_public_key_cache()?;
            let priv_key: recrypt::api::PrivateKey = recrypt::api::PrivateKey::new_from_slice(
                BASE64_STANDARD
                    .decode("bzb0Rlg0u7gx9wHuk1ppRI77OH/0ferXleenJ3Ag6Jg=")
                    .unwrap()
                    .as_slice(),
            )
            .unwrap();
            let dev_keys = recrypt::api::SigningKeypair::from_byte_slice(&[
                170, 222, 254, 96, 86, 46, 15, 233, 203, 170, 231, 41, 118, 13, 34, 45, 185, 234,
                6, 174, 28, 76, 100, 181, 86, 227, 113, 24, 4, 72, 162, 110, 16, 178, 40, 148, 87,
                243, 110, 163, 178, 75, 158, 100, 181, 167, 187, 6, 174, 69, 7, 78, 176, 97, 96,
                106, 28, 101, 179, 30, 150, 195, 24, 28,
            ])
            .unwrap();
            let wrong_device = DeviceContext::new(
                "account_id_2".try_into().unwrap(),
                23,
                priv_key.into(),
                DeviceSigningKeyPair::from(dev_keys),
            );
            let result =
                PublicKeyCache::deserialize_signed_public_key_cache(&wrong_device, &signed);
            assert!(result.is_err());
            Ok(())
        }
        #[test]
        fn signed_cache_too_short_fails() {
            let device = create_test_device_context();
            let result = PublicKeyCache::deserialize_signed_public_key_cache(&device, &[0u8; 32]);
            assert!(matches!(result, Err(IronOxideErr::WrongSizeError(_, _))));
        }
        #[test]
        fn signed_cache_exactly_64_bytes_no_payload_fails() {
            let device = create_test_device_context();
            let result = PublicKeyCache::deserialize_signed_public_key_cache(&device, &[0u8; 64]);
            // signature over empty payload won't match, or deserialization of empty bytes fails
            assert!(result.is_err());
        }
        #[test]
        // Build an IronOxide with a populated cache, export, then verify+deserialize
        fn export_then_deserialize_signed_roundtrip() -> Result<()> {
            let device = create_test_device_context();
            let recr = Recrypt::new();
            let (_, pubk) = recr.generate_key_pair().unwrap();

            let io = create_test_sdk()?;
            io.public_key_cache
                .user_keys()
                .pin()
                .insert(UserId::unsafe_from_string("user1".into()), pubk.into());

            let exported = io.export_public_key_cache().unwrap();
            let reimported =
                PublicKeyCache::deserialize_signed_public_key_cache(&device, &exported);
            assert!(reimported.is_ok());
            Ok(())
        }
    }
    #[tokio::test]
    async fn cache_lookup_empty_input_returns_empty() {
        let cache: HashMap<UserId, PublicKey> = HashMap::new();
        let (not_found, found) = get_keys_with_cache(&[], &cache, |_| async {
            panic!("fetch should not be called")
        })
        .await
        .unwrap();
        assert!(not_found.is_empty());
        assert!(found.is_empty());
    }
    #[tokio::test]
    async fn cache_lookup_all_cached_skips_fetch() {
        let recr = Recrypt::new();
        let (_, pubk) = recr.generate_key_pair().unwrap();
        let uid = UserId::unsafe_from_string("user1".into());

        let cache: HashMap<UserId, PublicKey> = HashMap::new();
        cache.pin().insert(uid.clone(), pubk.into());

        let (not_found, found) = get_keys_with_cache(&[uid.clone()], &cache, |_| async {
            panic!("fetch should not be called when fully cached")
        })
        .await
        .unwrap();

        assert!(not_found.is_empty());
        assert_eq!(found.len(), 1);
        assert_eq!(found[0].id, uid);
    }
    #[tokio::test]
    async fn cache_lookup_partial_cache_only_fetches_misses() {
        let recr = Recrypt::new();
        let (_, pub1) = recr.generate_key_pair().unwrap();
        let (_, pub2) = recr.generate_key_pair().unwrap();
        let cached_user = UserId::unsafe_from_string("cached".into());
        let uncached_user = UserId::unsafe_from_string("uncached".into());
        let io_pub2: PublicKey = pub2.into();

        let cache: HashMap<UserId, PublicKey> = HashMap::new();
        cache.pin().insert(cached_user.clone(), pub1.into());

        let pub2_clone = io_pub2.clone();
        let uncached_clone = uncached_user.clone();
        let (not_found, found) = get_keys_with_cache(
            &[cached_user.clone(), uncached_user.clone()],
            &cache,
            move |ids| async move {
                assert_eq!(ids.len(), 1, "should only try to fetch uncached ids");
                assert_eq!(ids[0], uncached_clone);
                let mut map = std::collections::HashMap::new();
                map.insert(uncached_clone, pub2_clone);
                Ok(map)
            },
        )
        .await
        .unwrap();

        assert!(not_found.is_empty());
        assert_eq!(found.len(), 2);
    }
    #[tokio::test]
    async fn cache_lookup_populates_cache_from_fetch() {
        let recr = Recrypt::new();
        let (_, pubk) = recr.generate_key_pair().unwrap();
        let uid = UserId::unsafe_from_string("user1".into());
        let io_pub: PublicKey = pubk.into();

        let cache: HashMap<UserId, PublicKey> = HashMap::new();
        assert!(cache.pin().get(&uid).is_none());

        let pub_clone = io_pub.clone();
        let uid_clone = uid.clone();
        get_keys_with_cache(&[uid.clone()], &cache, move |_| async move {
            let mut map = std::collections::HashMap::new();
            map.insert(uid_clone, pub_clone);
            Ok(map)
        })
        .await
        .unwrap();

        // cache should now contain the fetched key
        assert!(cache.pin().get(&uid).is_some());
    }
    #[tokio::test]
    async fn cache_lookup_fetch_returns_not_found() {
        let uid = UserId::unsafe_from_string("nonexistent".into());
        let cache: HashMap<UserId, PublicKey> = HashMap::new();

        let (not_found, found) = get_keys_with_cache(&[uid.clone()], &cache, |_| async {
            Ok(std::collections::HashMap::new())
        })
        .await
        .unwrap();

        assert_eq!(not_found.len(), 1);
        assert_eq!(not_found[0], uid);
        assert!(found.is_empty());
    }
    #[tokio::test]
    async fn cache_lookup_fetch_error_propagates() {
        let uid = UserId::unsafe_from_string("user1".into());
        let cache: HashMap<UserId, PublicKey> = HashMap::new();

        let result = get_keys_with_cache(&[uid], &cache, |_| async {
            Err(IronOxideErr::InitializeError(
                "simulated fetch failure".into(),
            ))
        })
        .await;

        assert!(result.is_err());
    }

    mod papaya_serde {
        use super::*;

        #[test]
        fn multi_user_multi_group_roundtrip() {
            let recrypt = Recrypt::new();
            let (_, creator_pubk) = recrypt.generate_key_pair().unwrap();
            let cache = PublicKeyCache::new(&creator_pubk.into());

            // insert multiple users
            for i in 0..5 {
                let (_, pubk) = recrypt.generate_key_pair().unwrap();
                cache
                    .user_keys()
                    .pin()
                    .insert(UserId::unsafe_from_string(format!("user_{i}")), pubk.into());
            }
            // insert multiple groups
            for i in 0..3 {
                let (_, pubk) = recrypt.generate_key_pair().unwrap();
                cache.group_keys().pin().insert(
                    GroupId::unsafe_from_string(format!("group_{i}")),
                    pubk.into(),
                );
            }

            let bytes = cache.serialize().unwrap();
            let deserialized = PublicKeyCache::deserialize(&bytes).unwrap();

            assert_eq!(deserialized.user_keys().len(), 5);
            assert_eq!(deserialized.group_keys().len(), 3);

            // verify specific entries survived
            for i in 0..5 {
                let uid = UserId::unsafe_from_string(format!("user_{i}"));
                let orig = cache.user_keys().pin().get(&uid).unwrap().clone();
                let deser = deserialized.user_keys().pin().get(&uid).unwrap().clone();
                assert_eq!(orig, deser);
            }
            for i in 0..3 {
                let gid = GroupId::unsafe_from_string(format!("group_{i}"));
                let orig = cache.group_keys().pin().get(&gid).unwrap().clone();
                let deser = deserialized.group_keys().pin().get(&gid).unwrap().clone();
                assert_eq!(orig, deser);
            }
        }

        #[test]
        fn same_key_different_ids_roundtrip() {
            let recrypt = Recrypt::new();
            let (_, shared_pubk) = recrypt.generate_key_pair().unwrap();
            let shared_pk: PublicKey = shared_pubk.into();

            let cache = PublicKeyCache::new(&shared_pk);
            cache.user_keys().pin().insert(
                UserId::unsafe_from_string("alice".into()),
                shared_pk.clone(),
            );
            cache
                .user_keys()
                .pin()
                .insert(UserId::unsafe_from_string("bob".into()), shared_pk.clone());

            let bytes = cache.serialize().unwrap();
            let deserialized = PublicKeyCache::deserialize(&bytes).unwrap();

            assert_eq!(deserialized.user_keys().len(), 2);
            let alice = deserialized
                .user_keys()
                .pin()
                .get(&UserId::unsafe_from_string("alice".into()))
                .unwrap()
                .clone();
            let bob = deserialized
                .user_keys()
                .pin()
                .get(&UserId::unsafe_from_string("bob".into()))
                .unwrap()
                .clone();
            assert_eq!(alice, bob);
            assert_eq!(alice, shared_pk);
        }

        #[test]
        fn truncated_bytes_fail_deserialization() {
            let recrypt = Recrypt::new();
            let (_, pubk) = recrypt.generate_key_pair().unwrap();

            let cache = PublicKeyCache::new(&pubk.into());
            cache
                .user_keys()
                .pin()
                .insert(UserId::unsafe_from_string("user1".into()), pubk.into());

            let bytes = cache.serialize().unwrap();
            // truncate to half the bytes
            let truncated = &bytes[..bytes.len() / 2];
            let result = PublicKeyCache::deserialize(truncated);
            assert!(result.is_err());
        }
    }
}