#[cfg(feature = "sync")]
use {
crate::{
data_api::{
WalletCommitmentTrees, WalletRead, WalletWrite,
chain::{
BlockCache, ChainState, CommitmentTreeRoot, error::Error as ChainError,
scan_cached_blocks,
},
scanning::{ScanPriority, ScanRange},
},
proto::service::{self, BlockId, compact_tx_streamer_client::CompactTxStreamerClient},
scanning::ScanError,
},
futures_util::TryStreamExt,
shardtree::error::ShardTreeError,
std::fmt,
subtle::ConditionallySelectable,
tonic::{
body::Body as TonicBody,
client::GrpcService,
codegen::{Body, Bytes, StdError},
},
tracing::{debug, info},
zcash_primitives::merkle_tree::HashSer,
zcash_protocol::consensus::{BlockHeight, Parameters},
};
#[cfg(all(feature = "sync", feature = "orchard"))]
use orchard::tree::MerkleHashOrchard;
#[cfg(all(feature = "sync", feature = "transparent-inputs"))]
use {
crate::wallet::WalletTransparentOutput,
::transparent::{
address::Script,
bundle::{OutPoint, TxOut},
},
zcash_keys::encoding::AddressCodec as _,
zcash_protocol::value::Zatoshis,
zcash_script::script,
};
#[cfg(feature = "sync-decryptor")]
pub mod decryptor;
#[cfg(feature = "sync")]
pub async fn run<P, ChT, CaT, DbT>(
client: &mut CompactTxStreamerClient<ChT>,
params: &P,
db_cache: &CaT,
db_data: &mut DbT,
batch_size: u32,
) -> Result<(), Error<CaT::Error, <DbT as WalletRead>::Error, <DbT as WalletCommitmentTrees>::Error>>
where
P: Parameters + Send + 'static,
ChT: GrpcService<TonicBody>,
ChT::Error: Into<StdError>,
ChT::ResponseBody: Body<Data = Bytes> + Send + 'static,
<ChT::ResponseBody as Body>::Error: Into<StdError> + Send,
CaT: BlockCache,
CaT::Error: std::error::Error + Send + Sync + 'static,
DbT: WalletWrite + WalletCommitmentTrees,
<DbT as WalletRead>::AccountId: ConditionallySelectable + Default + Send + Sync + 'static,
<DbT as WalletRead>::Error: std::error::Error + Send + Sync + 'static,
<DbT as WalletCommitmentTrees>::Error: std::error::Error + Send + Sync + 'static,
{
update_subtree_roots(client, db_data).await?;
while running(client, params, db_cache, db_data, batch_size).await? {}
Ok(())
}
#[cfg(feature = "sync")]
async fn running<P, ChT, CaT, DbT, TrErr>(
client: &mut CompactTxStreamerClient<ChT>,
params: &P,
db_cache: &CaT,
db_data: &mut DbT,
batch_size: u32,
) -> Result<bool, Error<CaT::Error, <DbT as WalletRead>::Error, TrErr>>
where
P: Parameters + Send + 'static,
ChT: GrpcService<TonicBody>,
ChT::Error: Into<StdError>,
ChT::ResponseBody: Body<Data = Bytes> + Send + 'static,
<ChT::ResponseBody as Body>::Error: Into<StdError> + Send,
CaT: BlockCache,
CaT::Error: std::error::Error + Send + Sync + 'static,
DbT: WalletWrite,
<DbT as WalletRead>::AccountId: ConditionallySelectable + Default + Send + Sync + 'static,
<DbT as WalletRead>::Error: std::error::Error + Send + Sync + 'static,
{
update_chain_tip(client, db_data).await?;
#[cfg(feature = "transparent-inputs")]
for account_id in db_data.get_account_ids().map_err(Error::Wallet)? {
let start_height = db_data
.utxo_query_height(account_id)
.map_err(Error::Wallet)?;
info!(
"Refreshing UTXOs for {:?} from height {}",
account_id, start_height,
);
refresh_utxos(params, client, db_data, account_id, start_height).await?;
}
let mut scan_ranges = db_data.suggest_scan_ranges().map_err(Error::Wallet)?;
let mut block_deletions = vec![];
loop {
match scan_ranges.first() {
Some(scan_range) if scan_range.priority() == ScanPriority::Verify => {
download_blocks(client, db_cache, scan_range).await?;
let chain_state =
download_chain_state(client, scan_range.block_range().start - 1).await?;
let scan_ranges_updated =
scan_blocks(params, db_cache, db_data, &chain_state, scan_range).await?;
block_deletions.push(db_cache.delete(scan_range.clone()));
if scan_ranges_updated {
scan_ranges = db_data.suggest_scan_ranges().map_err(Error::Wallet)?;
} else {
break;
}
}
_ => {
break;
}
}
}
let scan_ranges = db_data.suggest_scan_ranges().map_err(Error::Wallet)?;
debug!("Suggested ranges: {:?}", scan_ranges);
for scan_range in scan_ranges.into_iter().flat_map(|r| {
(0..).scan(r, |acc, _| {
if acc.is_empty() {
None
} else if let Some((cur, next)) = acc.split_at(acc.block_range().start + batch_size) {
*acc = next;
Some(cur)
} else {
let cur = acc.clone();
let end = acc.block_range().end;
*acc = ScanRange::from_parts(end..end, acc.priority());
Some(cur)
}
})
}) {
download_blocks(client, db_cache, &scan_range).await?;
let chain_state = download_chain_state(client, scan_range.block_range().start - 1).await?;
let scan_ranges_updated =
scan_blocks(params, db_cache, db_data, &chain_state, &scan_range).await?;
block_deletions.push(db_cache.delete(scan_range));
if scan_ranges_updated {
info!("Waiting for cached blocks to be deleted...");
for deletion in block_deletions {
deletion.await.map_err(Error::Cache)?;
}
return Ok(true);
}
}
info!("Waiting for cached blocks to be deleted...");
for deletion in block_deletions {
deletion.await.map_err(Error::Cache)?;
}
Ok(false)
}
#[cfg(feature = "sync")]
async fn download_subtree_roots<ChT, H>(
client: &mut CompactTxStreamerClient<ChT>,
protocol: service::ShieldedProtocol,
) -> Result<Vec<CommitmentTreeRoot<H>>, tonic::Status>
where
ChT: GrpcService<TonicBody>,
ChT::Error: Into<StdError>,
ChT::ResponseBody: Body<Data = Bytes> + Send + 'static,
<ChT::ResponseBody as Body>::Error: Into<StdError> + Send,
H: HashSer,
{
let mut request = service::GetSubtreeRootsArg::default();
request.set_shielded_protocol(protocol);
client
.get_subtree_roots(request)
.await?
.into_inner()
.and_then(|root| async move {
let root_hash = H::read(&root.root_hash[..])?;
Ok(CommitmentTreeRoot::from_parts(
BlockHeight::from_u32(root.completing_block_height as u32),
root_hash,
))
})
.try_collect()
.await
}
#[cfg(feature = "sync")]
async fn update_subtree_roots<ChT, DbT, CaErr, DbErr>(
client: &mut CompactTxStreamerClient<ChT>,
db_data: &mut DbT,
) -> Result<(), Error<CaErr, DbErr, <DbT as WalletCommitmentTrees>::Error>>
where
ChT: GrpcService<TonicBody>,
ChT::Error: Into<StdError>,
ChT::ResponseBody: Body<Data = Bytes> + Send + 'static,
<ChT::ResponseBody as Body>::Error: Into<StdError> + Send,
DbT: WalletCommitmentTrees,
<DbT as WalletCommitmentTrees>::Error: std::error::Error + Send + Sync + 'static,
{
let sapling_roots: Vec<CommitmentTreeRoot<sapling::Node>> =
download_subtree_roots(client, service::ShieldedProtocol::Sapling).await?;
info!("Sapling tree has {} subtrees", sapling_roots.len());
db_data
.put_sapling_subtree_roots(0, &sapling_roots)
.map_err(Error::WalletTrees)?;
#[cfg(feature = "orchard")]
{
let orchard_roots: Vec<CommitmentTreeRoot<MerkleHashOrchard>> =
download_subtree_roots(client, service::ShieldedProtocol::Orchard).await?;
info!("Orchard tree has {} subtrees", orchard_roots.len());
db_data
.put_orchard_subtree_roots(0, &orchard_roots)
.map_err(Error::WalletTrees)?;
let ironwood_roots: Vec<CommitmentTreeRoot<MerkleHashOrchard>> =
download_subtree_roots(client, service::ShieldedProtocol::Ironwood).await?;
info!("Ironwood tree has {} subtrees", ironwood_roots.len());
db_data
.put_ironwood_subtree_roots(0, &ironwood_roots)
.map_err(Error::WalletTrees)?;
}
Ok(())
}
#[cfg(feature = "sync")]
async fn update_chain_tip<ChT, DbT, CaErr, TrErr>(
client: &mut CompactTxStreamerClient<ChT>,
db_data: &mut DbT,
) -> Result<(), Error<CaErr, <DbT as WalletRead>::Error, TrErr>>
where
ChT: GrpcService<TonicBody>,
ChT::Error: Into<StdError>,
ChT::ResponseBody: Body<Data = Bytes> + Send + 'static,
<ChT::ResponseBody as Body>::Error: Into<StdError> + Send,
DbT: WalletWrite,
<DbT as WalletRead>::Error: std::error::Error + Send + Sync + 'static,
{
let tip_height: BlockHeight = client
.get_latest_block(service::ChainSpec::default())
.await?
.get_ref()
.height
.try_into()
.map_err(|_| Error::MisbehavingServer)?;
info!("Latest block height is {}", tip_height);
db_data
.update_chain_tip(tip_height)
.map_err(Error::Wallet)?;
Ok(())
}
#[cfg(feature = "sync")]
async fn download_blocks<ChT, CaT, DbErr, TrErr>(
client: &mut CompactTxStreamerClient<ChT>,
db_cache: &CaT,
scan_range: &ScanRange,
) -> Result<(), Error<CaT::Error, DbErr, TrErr>>
where
ChT: GrpcService<TonicBody>,
ChT::Error: Into<StdError>,
ChT::ResponseBody: Body<Data = Bytes> + Send + 'static,
<ChT::ResponseBody as Body>::Error: Into<StdError> + Send,
CaT: BlockCache,
CaT::Error: std::error::Error + Send + Sync + 'static,
{
info!("Fetching {}", scan_range);
let mut start = service::BlockId::default();
start.height = scan_range.block_range().start.into();
let mut end = service::BlockId::default();
end.height = (scan_range.block_range().end - 1).into();
let range = service::BlockRange {
start: Some(start),
end: Some(end),
pool_types: vec![],
};
let compact_blocks = client
.get_block_range(range)
.await?
.into_inner()
.try_collect::<Vec<_>>()
.await?;
db_cache
.insert(compact_blocks)
.await
.map_err(Error::Cache)?;
Ok(())
}
#[cfg(feature = "sync")]
async fn download_chain_state<ChT, CaErr, DbErr, TrErr>(
client: &mut CompactTxStreamerClient<ChT>,
block_height: BlockHeight,
) -> Result<ChainState, Error<CaErr, DbErr, TrErr>>
where
ChT: GrpcService<TonicBody>,
ChT::Error: Into<StdError>,
ChT::ResponseBody: Body<Data = Bytes> + Send + 'static,
<ChT::ResponseBody as Body>::Error: Into<StdError> + Send,
{
let tree_state = client
.get_tree_state(BlockId {
height: block_height.into(),
hash: vec![],
})
.await?;
tree_state
.into_inner()
.to_chain_state()
.map_err(|_| Error::MisbehavingServer)
}
#[cfg(feature = "sync")]
async fn scan_blocks<P, CaT, DbT, TrErr>(
params: &P,
db_cache: &CaT,
db_data: &mut DbT,
initial_chain_state: &ChainState,
scan_range: &ScanRange,
) -> Result<bool, Error<CaT::Error, <DbT as WalletRead>::Error, TrErr>>
where
P: Parameters + Send + 'static,
CaT: BlockCache,
CaT::Error: std::error::Error + Send + Sync + 'static,
DbT: WalletWrite,
<DbT as WalletRead>::AccountId: ConditionallySelectable + Default + Send + Sync + 'static,
<DbT as WalletRead>::Error: std::error::Error + Send + Sync + 'static,
{
info!("Scanning {}", scan_range);
let scan_result = scan_cached_blocks(
params,
db_cache,
db_data,
scan_range.block_range().start,
initial_chain_state,
scan_range.len(),
);
match scan_result {
Err(ChainError::Scan(err)) if err.is_continuity_error() => {
let rewind_height = err.at_height().saturating_sub(10);
info!(
"Chain reorg detected at {}, rewinding to {}",
err.at_height(),
rewind_height,
);
db_data
.truncate_to_height(rewind_height)
.map_err(Error::Wallet)?;
db_cache
.truncate(rewind_height)
.await
.map_err(Error::Cache)?;
Ok(true)
}
Ok(_) => {
let latest_ranges = db_data.suggest_scan_ranges().map_err(Error::Wallet)?;
Ok(if let Some(range) = latest_ranges.first() {
range.priority() > scan_range.priority()
} else {
false
})
}
Err(e) => Err(e.into()),
}
}
#[cfg(all(feature = "sync", feature = "transparent-inputs"))]
async fn refresh_utxos<P, ChT, DbT, CaErr, TrErr>(
params: &P,
client: &mut CompactTxStreamerClient<ChT>,
db_data: &mut DbT,
account_id: <DbT as WalletRead>::AccountId,
start_height: BlockHeight,
) -> Result<(), Error<CaErr, <DbT as WalletRead>::Error, TrErr>>
where
P: Parameters + Send + 'static,
ChT: GrpcService<TonicBody>,
ChT::Error: Into<StdError>,
ChT::ResponseBody: Body<Data = Bytes> + Send + 'static,
<ChT::ResponseBody as Body>::Error: Into<StdError> + Send,
DbT: WalletWrite,
<DbT as WalletRead>::Error: std::error::Error + Send + Sync + 'static,
{
let request = service::GetAddressUtxosArg {
addresses: db_data
.get_transparent_receivers(account_id, true, true)
.map_err(Error::Wallet)?
.into_keys()
.map(|addr| addr.encode(params))
.collect(),
start_height: start_height.into(),
max_entries: 0,
};
if request.addresses.is_empty() {
info!("{:?} has no transparent receivers", account_id);
} else {
client
.get_address_utxos_stream(request)
.await?
.into_inner()
.map_err(Error::Server)
.and_then(|reply| async move {
WalletTransparentOutput::from_parts(
OutPoint::new(
reply.txid[..]
.try_into()
.map_err(|_| Error::MisbehavingServer)?,
reply
.index
.try_into()
.map_err(|_| Error::MisbehavingServer)?,
),
TxOut::new(
Zatoshis::from_nonnegative_i64(reply.value_zat)
.map_err(|_| Error::MisbehavingServer)?,
Script(script::Code(reply.script)),
),
Some(
BlockHeight::try_from(reply.height)
.map_err(|_| Error::MisbehavingServer)?,
),
Some(account_id),
None,
None,
)
.ok_or(Error::MisbehavingServer)
})
.try_for_each(|output| {
let res = db_data.put_received_transparent_utxo(&output).map(|_| ());
async move { res.map_err(Error::Wallet) }
})
.await?;
}
Ok(())
}
#[cfg(feature = "sync")]
#[derive(Debug)]
#[non_exhaustive]
pub enum Error<CaErr, DbErr, TrErr> {
Cache(CaErr),
MisbehavingServer,
Scan(ScanError),
Server(tonic::Status),
Wallet(DbErr),
WalletTrees(ShardTreeError<TrErr>),
}
#[cfg(feature = "sync")]
impl<CaErr, DbErr, TrErr> fmt::Display for Error<CaErr, DbErr, TrErr>
where
CaErr: fmt::Display,
DbErr: fmt::Display,
TrErr: fmt::Display,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Error::Cache(e) => write!(f, "Error while interacting with block cache: {e}"),
Error::MisbehavingServer => write!(f, "lightwalletd server is misbehaving"),
Error::Scan(e) => write!(f, "Error while scanning blocks: {e}"),
Error::Server(e) => {
write!(f, "Error while communicating with lightwalletd server: {e}")
}
Error::Wallet(e) => write!(f, "Error while interacting with wallet database: {e}"),
Error::WalletTrees(e) => write!(
f,
"Error while interacting with wallet commitment trees: {e}"
),
}
}
}
#[cfg(feature = "sync")]
impl<CaErr, DbErr, TrErr> std::error::Error for Error<CaErr, DbErr, TrErr>
where
CaErr: std::error::Error,
DbErr: std::error::Error,
TrErr: std::error::Error,
{
}
#[cfg(feature = "sync")]
impl<CaErr, DbErr, TrErr> From<ChainError<DbErr, CaErr>> for Error<CaErr, DbErr, TrErr> {
fn from(e: ChainError<DbErr, CaErr>) -> Self {
match e {
ChainError::Wallet(e) => Error::Wallet(e),
ChainError::BlockSource(e) => Error::Cache(e),
ChainError::Scan(e) => Error::Scan(e),
}
}
}
#[cfg(feature = "sync")]
impl<CaErr, DbErr, TrErr> From<tonic::Status> for Error<CaErr, DbErr, TrErr> {
fn from(status: tonic::Status) -> Self {
Error::Server(status)
}
}
#[cfg(all(test, feature = "sync", feature = "orchard"))]
mod tests {
use std::{
convert::Infallible,
future::{Ready, ready},
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
task::{Context, Poll},
};
use tonic::{
body::Body as TonicBody,
codegen::{Service, http::Request, http::Response},
};
use zcash_protocol::consensus::Network;
use crate::{
data_api::testing::MockWalletDb,
proto::service::compact_tx_streamer_client::CompactTxStreamerClient,
};
use super::update_subtree_roots;
#[derive(Clone)]
struct CountingGrpcService(Arc<AtomicUsize>);
impl Service<Request<TonicBody>> for CountingGrpcService {
type Response = Response<TonicBody>;
type Error = Infallible;
type Future = Ready<Result<Self::Response, Self::Error>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, _request: Request<TonicBody>) -> Self::Future {
self.0.fetch_add(1, Ordering::Relaxed);
ready(Ok(Response::builder()
.header("content-type", "application/grpc")
.header("grpc-status", "0")
.body(TonicBody::empty())
.unwrap()))
}
}
#[test]
fn subtree_root_sync_requests_all_pools() {
let request_count = Arc::new(AtomicUsize::new(0));
let mut client =
CompactTxStreamerClient::new(CountingGrpcService(Arc::clone(&request_count)));
let mut wallet = MockWalletDb::new(Network::TestNetwork);
tokio::runtime::Runtime::new().unwrap().block_on(async {
update_subtree_roots::<_, _, Infallible, Infallible>(&mut client, &mut wallet)
.await
.unwrap();
});
assert_eq!(request_count.load(Ordering::Relaxed), 3);
}
}