blvm_protocol/utxo_commitments/

initial_sync.rs

//! Initial Sync Algorithm
//!
//! Implements the peer consensus initial sync algorithm:
//! 1. Discover diverse peers
//! 2. Determine checkpoint height
//! 3. Request UTXO sets from peers
//! 4. Find consensus
//! 5. Verify against block headers
//! 6. Download UTXO set

use crate::spam_filter::{SpamBreakdown, SpamFilter, SpamFilterConfig, SpamSummary, SpamType};
#[cfg(feature = "utxo-commitments")]
use crate::utxo_commitments::data_structures::{
    UtxoCommitment, UtxoCommitmentError, UtxoCommitmentResult,
};
#[cfg(feature = "utxo-commitments")]
use crate::utxo_commitments::merkle_tree::UtxoMerkleTree;
#[cfg(feature = "utxo-commitments")]
use crate::utxo_commitments::network_integration::UtxoCommitmentsNetworkClient;
#[cfg(feature = "utxo-commitments")]
use crate::utxo_commitments::peer_consensus::{ConsensusConfig, PeerConsensus, PeerInfo};
#[cfg(feature = "utxo-commitments")]
use blvm_consensus::types::{
    BlockHeader, Hash as HashType, Natural, OutPoint, Transaction, UTXO, UtxoSet,
};
#[cfg(feature = "utxo-commitments")]
/// Initial sync manager
pub struct InitialSync {
    peer_consensus: PeerConsensus,
    spam_filter: SpamFilter,
    // In real implementation: network_client: NetworkClient,
}

impl InitialSync {
    /// Create a new initial sync manager
    pub fn new(config: ConsensusConfig) -> Self {
        Self {
            peer_consensus: PeerConsensus::new(config),
            spam_filter: SpamFilter::new(),
        }
    }

    /// Create a new initial sync manager with custom spam filter config
    pub fn with_spam_filter(config: ConsensusConfig, spam_filter_config: SpamFilterConfig) -> Self {
        Self {
            peer_consensus: PeerConsensus::new(config),
            spam_filter: SpamFilter::with_config(spam_filter_config),
        }
    }

    /// Execute initial sync algorithm
    ///
    /// Performs the complete initial sync process:
    /// 1. Discover diverse peers
    /// 2. Determine checkpoint height
    /// 3. Request UTXO sets
    /// 4. Find consensus
    /// 5. Verify against headers
    /// 6. Return verified UTXO commitment
    pub async fn execute_initial_sync<C: UtxoCommitmentsNetworkClient>(
        &self,
        peers: &[(PeerInfo, String)],
        header_chain: &[BlockHeader],
        network_client: &C,
    ) -> UtxoCommitmentResult<UtxoCommitment> {
        // Step 1: Discover diverse peers (based on consensus-level PeerInfo)
        let all_infos: Vec<PeerInfo> = peers.iter().map(|(info, _)| info.clone()).collect();
        let diverse_infos = self.peer_consensus.discover_diverse_peers(all_infos);

        if diverse_infos.len() < self.peer_consensus.config.min_peers {
            return Err(UtxoCommitmentError::VerificationFailed(format!(
                "Insufficient diverse peers: got {}, need {}",
                diverse_infos.len(),
                self.peer_consensus.config.min_peers
            )));
        }

        // Re-attach peer IDs to the diverse set
        let diverse_with_ids: Vec<(PeerInfo, String)> = peers
            .iter()
            .filter(|(info, _)| {
                diverse_infos
                    .iter()
                    .any(|d| d.address == info.address && d.subnet == info.subnet)
            })
            .cloned()
            .collect();

        // Step 2: Determine checkpoint height from the verified header chain.
        // Peer tip queries (version/start_height aggregation) are not wired yet; the
        // header chain tip minus safety margin is the supported checkpoint path (REV-P-14).
        let checkpoint_height = if !header_chain.is_empty() {
            let tip = header_chain.len() as Natural - 1;
            tip.saturating_sub(self.peer_consensus.config.safety_margin)
        } else {
            return Err(UtxoCommitmentError::VerificationFailed(
                "No header chain available for checkpoint selection".to_string(),
            ));
        };

        // Get checkpoint block hash from header chain (unchanged)
        if checkpoint_height as usize >= header_chain.len() {
            return Err(UtxoCommitmentError::VerificationFailed(format!(
                "Checkpoint height {} exceeds header chain length {}",
                checkpoint_height,
                header_chain.len()
            )));
        }

        let checkpoint_header = &header_chain[checkpoint_height as usize];
        let checkpoint_hash = compute_block_hash(checkpoint_header);

        // Step 3: Request UTXO sets from diverse peers via the network client
        let peer_commitments = self
            .peer_consensus
            .request_utxo_sets(
                network_client,
                &diverse_with_ids,
                checkpoint_height,
                checkpoint_hash,
            )
            .await;

        // Step 4: Find consensus
        let consensus = self.peer_consensus.find_consensus(peer_commitments)?;

        // Step 5: Verify consensus commitment against block headers
        self.peer_consensus
            .verify_consensus_commitment(&consensus, header_chain)?;

        // Step 6: Per-UTXO proofs. An empty proof list is a failure, not a skip.
        #[cfg(feature = "utxo-proof-verification")]
        {
            let commitment =
                crate::utxo_commitments::peer_consensus::accept_commitment_proofs(&consensus)?;
            return Ok(commitment);
        }

        // Step 7: Return verified commitment
        Ok(consensus.commitment)
    }

    /// Complete sync from checkpoint to current tip
    ///
    /// Syncs forward from checkpoint using FULL blocks with complete validation.
    /// Fully validates all transactions (signatures, scripts) before updating UTXO set.
    ///
    /// # Arguments
    ///
    /// * `utxo_tree` - UTXO Merkle tree to update incrementally
    /// * `checkpoint_height` - Starting height (checkpoint)
    /// * `current_tip` - Ending height (current chain tip)
    /// * `network_client` - Network client for requesting blocks
    /// * `get_block_hash` - Function to get block hash for a given height
    /// * `peer_id` - Peer ID to request blocks from
    /// * `network` - Network type (Mainnet, Testnet, Regtest)
    /// * `network_time` - Current network time (Unix timestamp)
    /// * `recent_headers` - Recent block headers for median time-past calculation
    /// * `checkpoint_utxo_set` - Full UTXO set at checkpoint (required for validation).
    ///   If None, starts with empty set (cannot verify checkpoint commitment until end)
    ///
    /// # Implementation
    ///
    /// 1. Requests FULL blocks from checkpoint+1 to tip
    /// 2. For each full block:
    ///    - Fully validates block with connect_block() (signatures, scripts, all consensus rules)
    ///    - Updates UTXO set from validated result
    ///    - Updates UTXO tree from validated UTXO set
    ///    - Verifies commitment matches computed root
    /// 3. Updates UTXO tree incrementally after validation
    ///
    /// **Security**: All transactions are cryptographically verified before UTXO set update.
    #[allow(clippy::too_many_arguments)]
    pub async fn complete_sync_from_checkpoint<C, F, Fut>(
        &self,
        utxo_tree: &mut UtxoMerkleTree,
        checkpoint_height: Natural,
        current_tip: Natural,
        network_client: &C,
        get_block_hash: F,
        peer_id: &str,
        network: crate::types::Network,
        network_time: u64,
        recent_headers: Option<&[BlockHeader]>,
        checkpoint_utxo_set: Option<UtxoSet>,
    ) -> UtxoCommitmentResult<()>
    where
        C: UtxoCommitmentsNetworkClient,
        F: Fn(Natural) -> Fut,
        Fut: std::future::Future<Output = UtxoCommitmentResult<HashType>>,
    {
        use crate::block::connect_block;

        // Start with checkpoint UTXO set, or empty if not provided
        // Note: If empty, we cannot verify checkpoint commitment until we've built the full set
        let mut utxo_set: UtxoSet = checkpoint_utxo_set.unwrap_or_default();

        // Process blocks incrementally from checkpoint+1 to current tip
        for height in checkpoint_height + 1..=current_tip {
            // Get block hash for this height
            let block_hash = get_block_hash(height).await?;

            // Request FULL block (not filtered) from network peer
            // This includes all transactions with witnesses for complete validation
            let full_block = network_client
                .request_full_block(peer_id, block_hash)
                .await?;

            // Verify block header height matches expected height
            if full_block.block.header.timestamp == 0 {
                return Err(UtxoCommitmentError::VerificationFailed(format!(
                    "Invalid block header at height {height}"
                )));
            }

            let context = crate::block::block_validation_context_for_connect_ibd(
                recent_headers,
                network_time,
                network,
            );
            let (validation_result, new_utxo_set, _undo_log) = connect_block(
                &full_block.block,
                &full_block.witnesses,
                utxo_set.clone(),
                height,
                &context,
            )
            .map_err(|e| {
                UtxoCommitmentError::VerificationFailed(format!(
                    "connect_block failed at height {height}: {e}"
                ))
            })?;

            // Reject if validation failed
            if !matches!(
                validation_result,
                blvm_consensus::types::ValidationResult::Valid
            ) {
                return Err(UtxoCommitmentError::VerificationFailed(format!(
                    "Block validation failed at height {height}: {validation_result:?}"
                )));
            }

            // Update UTXO tree from validated UTXO set
            // This ensures the tree matches the cryptographically verified state
            // Pass old utxo_set to detect removals efficiently
            let old_utxo_set = utxo_set.clone();
            utxo_tree.update_from_utxo_set(&new_utxo_set, &old_utxo_set)?;

            // Generate commitment from validated UTXO tree
            let computed_block_hash = compute_block_hash(&full_block.block.header);
            let computed_commitment = utxo_tree.generate_commitment(computed_block_hash, height);

            // Verify commitment supply matches expected
            use blvm_consensus::economic::total_supply;
            let expected_supply = total_supply(height) as u64;
            if computed_commitment.total_supply != expected_supply {
                return Err(UtxoCommitmentError::VerificationFailed(format!(
                    "Supply mismatch at height {}: computed {}, expected {}",
                    height, computed_commitment.total_supply, expected_supply
                )));
            }

            // Note: Commitment root, height, and block hash are verified implicitly
            // by the fact that we computed them from the validated UTXO set

            // Update utxo_set for next iteration
            utxo_set = new_utxo_set;
        }

        Ok(())
    }

    /// Process a filtered block and update UTXO set
    ///
    /// Takes a block with transactions (already filtered or to be filtered),
    /// applies spam filter, updates UTXO set, and verifies commitment.
    ///
    /// **Critical**: This function processes ALL transactions to remove spent inputs,
    /// but only adds non-spam outputs to the UTXO tree. This ensures UTXO set consistency:
    /// - Spam transactions that spend non-spam inputs will still remove those inputs
    /// - Only non-spam outputs are added to the tree (bandwidth savings)
    /// - UTXO set remains consistent with actual blockchain state
    ///
    /// Note: This function applies transactions to the UTXO tree for commitment
    /// purposes. Full signature verification should be done during block validation
    /// before calling this function. This function assumes transactions are already
    /// validated.
    pub fn process_filtered_block(
        &self,
        utxo_tree: &mut UtxoMerkleTree,
        block_height: Natural,
        block_transactions: &[Transaction],
    ) -> UtxoCommitmentResult<(SpamSummary, HashType)> {
        use blvm_consensus::transaction::is_coinbase;

        let mut spam_summary = SpamSummary {
            filtered_count: 0,
            filtered_size: 0,
            by_type: SpamBreakdown::default(),
        };

        // Process ALL transactions (including spam) to remove spent inputs
        // This is critical for UTXO set consistency: even spam transactions must
        // remove their spent inputs from the tree.
        for tx in block_transactions {
            // Check if transaction is spam (for output filtering)
            let spam_result = self.spam_filter.is_spam(tx);
            let is_spam = spam_result.is_spam;

            // Update spam summary
            if is_spam {
                spam_summary.filtered_count += 1;
                spam_summary.filtered_size +=
                    blvm_consensus::transaction::calculate_transaction_size(tx) as u64;

                // Update breakdown
                for spam_type in &spam_result.detected_types {
                    match spam_type {
                        SpamType::Ordinals => {
                            spam_summary.by_type.ordinals += 1;
                        }
                        SpamType::Dust => {
                            spam_summary.by_type.dust += 1;
                        }
                        SpamType::BRC20 => {
                            spam_summary.by_type.brc20 += 1;
                        }
                        SpamType::LargeWitness => {
                            spam_summary.by_type.ordinals += 1; // Count as Ordinals
                        }
                        SpamType::LowFeeRate => {
                            spam_summary.by_type.dust += 1; // Count as suspicious
                        }
                        SpamType::HighSizeValueRatio => {
                            spam_summary.by_type.ordinals += 1; // Count as Ordinals
                        }
                        SpamType::ManySmallOutputs => {
                            spam_summary.by_type.dust += 1; // Count as dust-like
                        }
                        SpamType::UnexecIf
                        | SpamType::NullDataOp13
                        | SpamType::NullDataMagic
                        | SpamType::DataLikeMs => {
                            spam_summary.by_type.ordinals += 1;
                        }
                        SpamType::NotSpam => {}
                    }
                }
            }

            // Compute transaction ID for proper outpoint creation
            let tx_id = compute_tx_id(tx);

            // CRITICAL: Remove spent inputs from ALL transactions (including spam)
            // This ensures UTXO set consistency even when spam transactions spend non-spam inputs
            if !is_coinbase(tx) {
                for input in &tx.inputs {
                    // Get the UTXO first (needed for remove to update tracking)
                    match utxo_tree.get(&input.prevout) {
                        Ok(Some(utxo)) => {
                            // Remove the UTXO (even if transaction is spam)
                            if let Err(e) = utxo_tree.remove(&input.prevout, &utxo) {
                                return Err(UtxoCommitmentError::TransactionApplication(format!(
                                    "Failed to remove spent input: {e:?}"
                                )));
                            }
                        }
                        Ok(None) => {
                            // UTXO doesn't exist - this might be valid if it was already spent
                            // or invalid if the transaction wasn't properly validated
                            // Continue but log - this should be validated before calling
                        }
                        Err(e) => {
                            return Err(UtxoCommitmentError::TransactionApplication(format!(
                                "Failed to get UTXO for removal: {e:?}"
                            )));
                        }
                    }
                }
            }

            // Only add outputs from non-spam transactions
            // This provides bandwidth savings while maintaining UTXO set consistency
            if !is_spam {
                for (i, output) in tx.outputs.iter().enumerate() {
                    let outpoint = OutPoint {
                        hash: tx_id,
                        index: i as u32,
                    };

                    let utxo = UTXO {
                        value: output.value,
                        script_pubkey: output.script_pubkey.as_slice().into(),
                        height: block_height,
                        is_coinbase: is_coinbase(tx),
                    };

                    if let Err(e) = utxo_tree.insert(outpoint, utxo) {
                        return Err(UtxoCommitmentError::TransactionApplication(format!(
                            "Failed to add output: {e:?}"
                        )));
                    }
                }
            }
            // Spam transaction outputs are skipped (not added to tree)
        }

        // Return summary and new root
        let root = utxo_tree.root();

        Ok((spam_summary, root))
    }
}

/// Update UTXO commitments after block connection
///
/// This function should be called after successfully connecting a block
/// to keep UTXO commitments synchronized with the blockchain state.
///
/// # Arguments
///
/// * `utxo_tree` - Mutable reference to the UTXO Merkle tree
/// * `block` - The block that was just connected
/// * `block_height` - Height of the connected block
/// * `spam_filter` - Optional spam filter (if None, all transactions are included)
///
/// # Returns
///
/// Returns the new Merkle root hash of the UTXO tree.
///
/// # Example
///
/// ```rust
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
/// use blvm_consensus::block::connect_block;
/// use blvm_protocol::utxo_commitments::{UtxoMerkleTree, update_commitments_after_block};
/// use blvm_consensus::spam_filter::SpamFilter;
/// use blvm_consensus::types::{Network, ValidationResult};
///
/// # let block = blvm_consensus::types::Block {
/// #     header: blvm_consensus::types::BlockHeader {
/// #         version: 1, prev_block_hash: [0; 32], merkle_root: [0; 32],
/// #         timestamp: 1234567890, bits: 0x1d00ffff, nonce: 0,
/// #     },
/// #     transactions: vec![].into(),
/// # };
/// # let witnesses = vec![];
/// # let utxo_set = blvm_consensus::types::UtxoSet::new();
/// # let height = 0;
/// # let mut utxo_tree = UtxoMerkleTree::new()?;
/// let (result, new_utxo_set, _) = connect_block(&block, &witnesses, utxo_set, height, None, Network::Regtest)?;
/// if matches!(result, ValidationResult::Valid) {
///     let spam_filter = SpamFilter::new();
///     let root = update_commitments_after_block(
///         &mut utxo_tree,
///         &block,
///         height,
///         Some(&spam_filter),
///     )?;
///     println!("New UTXO commitment root: {:?}", root);
/// }
/// # Ok(())
/// # }
/// ```
#[cfg(feature = "utxo-commitments")]
pub fn update_commitments_after_block(
    utxo_tree: &mut UtxoMerkleTree,
    block: &crate::types::Block,
    block_height: Natural,
    spam_filter: Option<&SpamFilter>,
) -> UtxoCommitmentResult<HashType> {
    use blvm_consensus::block::calculate_tx_id;
    use blvm_consensus::transaction::is_coinbase;

    // If spam filter is provided, use filtered processing
    if let Some(filter) = spam_filter {
        let initial_sync = InitialSync {
            peer_consensus: crate::utxo_commitments::peer_consensus::PeerConsensus::new(
                crate::utxo_commitments::peer_consensus::ConsensusConfig::default(),
            ),
            spam_filter: filter.clone(),
        };
        let (_, root) =
            initial_sync.process_filtered_block(utxo_tree, block_height, &block.transactions)?;
        Ok(root)
    } else {
        // No spam filter: process all transactions normally
        for tx in &block.transactions {
            let tx_id = calculate_tx_id(tx);

            // Remove spent inputs (except coinbase)
            if !is_coinbase(tx) {
                for input in &tx.inputs {
                    // Get the UTXO first (needed for remove)
                    match utxo_tree.get(&input.prevout) {
                        Ok(Some(utxo)) => {
                            utxo_tree.remove(&input.prevout, &utxo)?;
                        }
                        Ok(None) => {
                            // UTXO doesn't exist - might be invalid or already spent
                            // Continue but this should have been caught during validation
                        }
                        Err(e) => {
                            return Err(UtxoCommitmentError::TransactionApplication(format!(
                                "Failed to get UTXO for removal: {e:?}"
                            )));
                        }
                    }
                }
            }

            // Add new outputs
            for (i, output) in tx.outputs.iter().enumerate() {
                let outpoint = blvm_consensus::types::OutPoint {
                    hash: tx_id,
                    index: i as u32,
                };

                let utxo = blvm_consensus::types::UTXO {
                    value: output.value,
                    script_pubkey: output.script_pubkey.as_slice().into(),
                    height: block_height,
                    is_coinbase: is_coinbase(tx),
                };

                utxo_tree.insert(outpoint, utxo)?;
            }
        }

        Ok(utxo_tree.root())
    }
}

/// Compute transaction ID (txid) using Bitcoin's standard double SHA256
///
/// Transaction ID is computed as: SHA256(SHA256(serialized_tx))
/// where serialized_tx is the transaction in Bitcoin wire format (non-SegWit format).
///
/// Note: For SegWit transactions, the txid still uses the non-witness serialization
/// (witness data is excluded from txid calculation).
///
/// This matches consensus transaction ID computation exactly.
fn compute_tx_id(tx: &Transaction) -> HashType {
    use crate::serialization::transaction::serialize_transaction;
    use sha2::{Digest, Sha256};

    // Serialize transaction to Bitcoin wire format (non-SegWit format for txid)
    let serialized = serialize_transaction(tx);

    // Double SHA256 (Bitcoin standard for transaction IDs)
    let first_hash = Sha256::digest(&serialized);
    let second_hash = Sha256::digest(first_hash);

    // Convert to HashType [u8; 32]
    let mut txid = [0u8; 32];
    txid.copy_from_slice(&second_hash);

    txid
}

/// Compute block header hash (double SHA256)
fn compute_block_hash(header: &BlockHeader) -> HashType {
    use sha2::{Digest, Sha256};

    let mut bytes = Vec::with_capacity(80);
    bytes.extend_from_slice(&header.version.to_le_bytes());
    bytes.extend_from_slice(&header.prev_block_hash);
    bytes.extend_from_slice(&header.merkle_root);
    bytes.extend_from_slice(&header.timestamp.to_le_bytes());
    bytes.extend_from_slice(&header.bits.to_le_bytes());
    bytes.extend_from_slice(&header.nonce.to_le_bytes());

    let first_hash = Sha256::digest(&bytes);
    let second_hash = Sha256::digest(first_hash);

    let mut hash = [0u8; 32];
    hash.copy_from_slice(&second_hash);
    hash
}