use std::time::Duration;
use super::MultisigClient;
use super::proposal_note_import::{NoteImportOutcome, NoteImportSource, NoteImportStatus};
use super::public_note_backfill::{PublicBackfillOptions, PublicBackfillReport};
use super::recovery::TransportRecoveryReport;
use crate::error::{MultisigError, Result, error_chain};
pub const PRE_SWITCH_IMPORT_TIMEOUT: Duration = Duration::from_secs(30);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct NoteRecoveryOptions {
pub transport_drain: bool,
pub proposal_import: bool,
pub public_backfill: bool,
pub backfill: PublicBackfillOptions,
pub sync_after: bool,
}
impl Default for NoteRecoveryOptions {
fn default() -> Self {
Self {
transport_drain: true,
proposal_import: true,
public_backfill: true,
backfill: PublicBackfillOptions::default(),
sync_after: true,
}
}
}
impl NoteRecoveryOptions {
pub(crate) fn for_guardian_switch() -> Self {
Self {
transport_drain: false,
proposal_import: true,
public_backfill: false,
backfill: PublicBackfillOptions::default(),
sync_after: false,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RecoveryStep {
TransportDrain,
ProposalImport,
PublicBackfill,
Sync,
}
impl RecoveryStep {
pub fn as_str(&self) -> &'static str {
match self {
Self::TransportDrain => "transport-drain",
Self::ProposalImport => "proposal-import",
Self::PublicBackfill => "public-backfill",
Self::Sync => "sync",
}
}
}
impl std::fmt::Display for RecoveryStep {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RecoveryStepProblem {
pub step: RecoveryStep,
pub reason: String,
pub retryable: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NoteRecoveryReport {
pub transport: Option<TransportRecoveryReport>,
pub proposal_import: Option<Vec<NoteImportOutcome>>,
pub backfill: Option<PublicBackfillReport>,
pub problems: Vec<RecoveryStepProblem>,
pub synced: bool,
pub imported: usize,
pub retryable: bool,
}
impl MultisigClient {
pub async fn recover_notes(
&mut self,
options: Option<NoteRecoveryOptions>,
) -> Result<NoteRecoveryReport> {
let options = options.unwrap_or_default();
let account_id = self.require_account()?.id();
if let (Some(from), Some(to)) = (options.backfill.from_block, options.backfill.to_block)
&& from > to
{
return Err(MultisigError::InvalidConfig(format!(
"backfill range is inverted: from_block {from} > to_block {to}"
)));
}
let mut report = NoteRecoveryReport {
transport: None,
proposal_import: None,
backfill: None,
problems: Vec::new(),
synced: false,
imported: 0,
retryable: false,
};
if options.transport_drain {
match self.drain_private_note_backlog().await {
Ok(drain) => report.transport = Some(drain),
Err(e) => report.problems.push(RecoveryStepProblem {
step: RecoveryStep::TransportDrain,
reason: error_chain(&e),
retryable: false,
}),
}
}
if options.proposal_import {
match self.list_proposals_isolating_failures().await {
Ok((proposals, skipped)) => {
let mut outcomes: Vec<NoteImportOutcome> = skipped
.into_iter()
.map(|(identifier, reason)| NoteImportOutcome {
identifier,
source: NoteImportSource::Proposal,
status: NoteImportStatus::Invalid,
retryable: false,
reason: Some(reason),
})
.collect();
outcomes.extend(self.import_notes_from_proposals(&proposals).await);
report.proposal_import = Some(outcomes);
}
Err(e) => report.problems.push(RecoveryStepProblem {
step: RecoveryStep::ProposalImport,
reason: format!("failed to list pending proposals: {}", error_chain(&e)),
retryable: true,
}),
}
}
if options.public_backfill {
let synced_for_backfill = match self.miden_client.sync_chain().await {
Ok(_) => true,
Err(e) => {
report.problems.push(RecoveryStepProblem {
step: RecoveryStep::PublicBackfill,
reason: format!(
"failed to sync the chain state the backfill imports against: {}",
error_chain(&e)
),
retryable: true,
});
false
}
};
if synced_for_backfill {
match self
.backfill_public_notes_by_tag(account_id, Some(options.backfill))
.await
{
Ok(backfill) => report.backfill = Some(backfill),
Err(e) => {
let retryable = !matches!(e, MultisigError::InvalidConfig(_));
report.problems.push(RecoveryStepProblem {
step: RecoveryStep::PublicBackfill,
reason: error_chain(&e),
retryable,
});
}
}
}
}
if options.sync_after {
match self.sync().await {
Ok(()) => report.synced = true,
Err(e) => report.problems.push(RecoveryStepProblem {
step: RecoveryStep::Sync,
reason: format!("recovery imports succeeded but the verifying sync failed; rerun a sync: {}", error_chain(&e)),
retryable: true,
}),
}
}
let imported_outcomes = |outcomes: &[NoteImportOutcome]| {
outcomes
.iter()
.filter(|o| o.status == NoteImportStatus::Imported)
.count()
};
report.imported = report.transport.as_ref().map_or(0, |t| t.imported)
+ report
.proposal_import
.as_ref()
.map_or(0, |o| imported_outcomes(o))
+ report
.backfill
.as_ref()
.map_or(0, |b| imported_outcomes(&b.outcomes));
report.retryable = report.problems.iter().any(|p| p.retryable)
|| report.transport.as_ref().is_some_and(|t| t.retryable)
|| report
.proposal_import
.as_ref()
.is_some_and(|o| o.iter().any(|o| o.retryable))
|| report.backfill.as_ref().is_some_and(|b| b.retryable);
Ok(report)
}
pub async fn preserve_pre_switch_proposal_notes(&mut self) -> Option<NoteRecoveryReport> {
let flow = self.recover_notes(Some(NoteRecoveryOptions::for_guardian_switch()));
match tokio::time::timeout(PRE_SWITCH_IMPORT_TIMEOUT, flow).await {
Err(_elapsed) => {
tracing::warn!(
timeout_secs = PRE_SWITCH_IMPORT_TIMEOUT.as_secs(),
"pre-switch proposal-note import timed out and was cancelled; notes \
embedded in pending proposals may be unrecoverable after the \
guardian switch"
);
None
}
Ok(Ok(report)) => {
for problem in &report.problems {
tracing::warn!(
step = problem.step.as_str(),
reason = %problem.reason,
"pre-switch proposal-note import did not finish; notes embedded \
in pending proposals may be unrecoverable after the guardian \
switch"
);
}
for outcome in report.proposal_import.iter().flatten().filter(|o| {
matches!(
o.status,
NoteImportStatus::Invalid | NoteImportStatus::Failed
)
}) {
tracing::warn!(
identifier = %outcome.identifier,
status = outcome.status.as_str(),
reason = outcome.reason.as_deref().unwrap_or("unknown"),
"pre-switch import could not preserve an embedded note; it may \
be unrecoverable after the guardian switch"
);
}
Some(report)
}
Ok(Err(e)) => {
tracing::warn!(
error = %error_chain(&e),
"pre-switch proposal-note import could not run; notes embedded in \
pending proposals may be unrecoverable after the guardian switch"
);
None
}
}
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use miden_client::testing::mock::MockRpcApi;
use miden_protocol::block::BlockNumber;
use miden_protocol::note::NoteType;
use super::super::recovery::TransportRecoveryStatus;
use super::*;
use crate::account::MultisigAccount;
use crate::client::public_note_backfill::PublicBackfillOptions;
use crate::client::test_support::{
add_to_transport, mock_transport, offline_client, offline_client_with_node, p2id_note_for,
test_wallet,
};
async fn load_account(client: &mut MultisigClient, seed: u8) {
let account = test_wallet(seed);
client.add_or_update_account(&account, true).await.unwrap();
client.account = Some(MultisigAccount::new(account));
}
#[tokio::test]
async fn recover_notes_requires_a_loaded_account() {
let dir = tempfile::tempdir().unwrap();
let mut client = offline_client(dir.path(), None).await;
assert!(client.recover_notes(None).await.is_err());
}
#[tokio::test]
async fn recover_notes_rejects_an_inverted_backfill_range() {
let dir = tempfile::tempdir().unwrap();
let mut client = offline_client(dir.path(), None).await;
load_account(&mut client, 31).await;
let err = client
.recover_notes(Some(NoteRecoveryOptions {
backfill: PublicBackfillOptions {
from_block: Some(BlockNumber::from(5u32)),
to_block: Some(BlockNumber::from(1u32)),
},
..Default::default()
}))
.await
.expect_err("inverted range must error");
assert!(err.to_string().contains("inverted"));
}
#[tokio::test]
async fn recover_notes_continues_past_failing_steps() {
let dir = tempfile::tempdir().unwrap();
let (node, api) = mock_transport();
let mut client = offline_client(dir.path(), Some(api)).await;
load_account(&mut client, 32).await;
let account = client.account.as_ref().unwrap().inner().clone();
add_to_transport(&node, p2id_note_for(&account, 1, NoteType::Private));
let report = client.recover_notes(None).await.unwrap();
let transport = report.transport.expect("drain ran");
assert_eq!(transport.status, TransportRecoveryStatus::Completed);
assert_eq!(transport.imported, 1);
assert_eq!(report.proposal_import, None);
assert_eq!(report.backfill, None);
assert!(!report.synced);
assert_eq!(report.imported, 1);
assert!(report.retryable);
let steps: Vec<RecoveryStep> = report.problems.iter().map(|p| p.step).collect();
assert_eq!(
steps,
vec![
RecoveryStep::ProposalImport,
RecoveryStep::PublicBackfill,
RecoveryStep::Sync,
]
);
assert!(report.problems.iter().all(|p| p.retryable));
}
#[tokio::test]
async fn recover_notes_runs_only_the_selected_strategies() {
let dir = tempfile::tempdir().unwrap();
let (_node, api) = mock_transport();
let mut client = offline_client(dir.path(), Some(api)).await;
load_account(&mut client, 33).await;
let report = client
.recover_notes(Some(NoteRecoveryOptions {
transport_drain: true,
proposal_import: false,
public_backfill: false,
backfill: PublicBackfillOptions::default(),
sync_after: false,
}))
.await
.unwrap();
let transport = report.transport.expect("drain ran");
assert_eq!(transport.status, TransportRecoveryStatus::Completed);
assert_eq!(report.proposal_import, None);
assert_eq!(report.backfill, None);
assert!(report.problems.is_empty());
assert!(!report.synced);
assert_eq!(report.imported, 0);
assert!(!report.retryable);
}
#[tokio::test]
async fn preserve_pre_switch_proposal_notes_runs_only_the_import_and_never_errors() {
let dir = tempfile::tempdir().unwrap();
let (node, api) = mock_transport();
let mut client = offline_client(dir.path(), Some(api)).await;
load_account(&mut client, 35).await;
let account = client.account.as_ref().unwrap().inner().clone();
add_to_transport(&node, p2id_note_for(&account, 1, NoteType::Private));
let report = client
.preserve_pre_switch_proposal_notes()
.await
.expect("best-effort flow reports instead of erroring");
assert_eq!(
report.transport, None,
"transport drain must not run on the switch path"
);
assert_eq!(
report.backfill, None,
"public backfill must not run on the switch path"
);
assert!(!report.synced, "the switch flow owns its own syncs");
assert_eq!(report.imported, 0);
let steps: Vec<RecoveryStep> = report.problems.iter().map(|p| p.step).collect();
assert_eq!(steps, vec![RecoveryStep::ProposalImport]);
assert!(report.retryable);
}
#[tokio::test]
async fn recover_notes_reports_backfill_and_unavailable_transport() {
let dir = tempfile::tempdir().unwrap();
let mut client =
offline_client_with_node(dir.path(), Arc::new(MockRpcApi::default())).await;
load_account(&mut client, 34).await;
let report = client
.recover_notes(Some(NoteRecoveryOptions {
proposal_import: false,
sync_after: false,
..Default::default()
}))
.await
.unwrap();
let transport = report.transport.expect("drain ran");
assert_eq!(transport.status, TransportRecoveryStatus::Unavailable);
let backfill = report.backfill.expect("backfill ran");
assert_eq!(backfill.scanned_from, 0);
assert!(backfill.uncovered.is_empty());
assert!(report.problems.is_empty());
assert_eq!(report.imported, 0);
assert!(!report.retryable);
}
}