zebra-utils 2.0.0

#!/usr/bin/env bash

set -uo pipefail

# Sends a `zcash-cli` command to a Zebra and zcashd instance,
# and compares the results.
#
# Uses the configured `zcash-cli` RPC port,
# and the `zebrad` port supplied on the command-line.

function usage()
{
    echo "Usage:"
    echo "$0 zebra-rpc-port rpc-name [rpc-args... ]"
}

# Override the commands used by this script using these environmental variables:
ZCASH_CLI="${ZCASH_CLI:-zcash-cli}"
DIFF="${DIFF:-diff --unified --color=always}"
JQ="${JQ:-jq}"
# Zcashd authentication modes:
# - Use `-rpccookiefile=your/cookie/file` for a cookie file.
# - Use `-rpcpassword=your-password` for a password.
ZCASHD_EXTRA_ARGS="${ZCASHD_EXTRA_ARGS:-}"
# Zebrad authentication modes:
# - Use `-rpccookiefile=your/cookie/file` for a cookie file.
ZEBRAD_EXTRA_ARGS="${ZEBRAD_EXTRA_ARGS:-}"
# We show this many lines of data, removing excess lines from the middle or end of the output.
OUTPUT_DATA_LINE_LIMIT="${OUTPUT_DATA_LINE_LIMIT:-40}"
# When checking different mempools, we show this many different transactions.
MEMPOOL_TX_LIMIT="${MEMPOOL_TX_LIMIT:-5}"


if [ $# -lt 2 ]; then
    usage
    exit 1
fi

ZEBRAD_RPC_PORT=$1
shift

echo "Using '$($ZCASH_CLI -version | head -1)' via command '$ZCASH_CLI'."
echo "Using bash shell '$BASH_VERSION' launched from '$SHELL'."

echo

# Use an easily identified temp directory name,
# but fall back to the default temp name if `mktemp` does not understand `--suffix`.
ZCASH_RPC_TMP_DIR=$(mktemp --suffix=.rpc-diff -d 2>/dev/null || mktemp -d)

ZEBRAD_RELEASE_INFO="$ZCASH_RPC_TMP_DIR/first-node-check-getinfo.json"
ZCASHD_RELEASE_INFO="$ZCASH_RPC_TMP_DIR/second-node-check-getinfo.json"

echo "Checking first node release info..."
$ZCASH_CLI $ZEBRAD_EXTRA_ARGS -rpcport="$ZEBRAD_RPC_PORT" getinfo > "$ZEBRAD_RELEASE_INFO"

ZEBRAD_NAME=$(cat "$ZEBRAD_RELEASE_INFO" | grep '"subversion"' | cut -d: -f2 | cut -d/ -f2 | \
             tr 'A-Z' 'a-z' | sed 's/magicbean/zcashd/ ; s/zebra$/zebrad/')
ZEBRAD_VERSION=$(cat "$ZEBRAD_RELEASE_INFO" | grep '"build"' | cut -d: -f2 | cut -d'"' -f2 | \
                     tr 'A-Z' 'a-z')
ZEBRAD="$ZEBRAD_NAME $ZEBRAD_VERSION"

echo "Checking second node release info..."
$ZCASH_CLI $ZCASHD_EXTRA_ARGS getinfo > "$ZCASHD_RELEASE_INFO"

ZCASHD_NAME=$(cat "$ZCASHD_RELEASE_INFO" | grep '"subversion"' | cut -d: -f2 | cut -d/ -f2 | \
             tr 'A-Z' 'a-z' | sed 's/magicbean/zcashd/ ; s/zebra$/zebrad/')
ZCASHD_VERSION=$(cat "$ZCASHD_RELEASE_INFO" | grep '"build"' | cut -d: -f2 | cut -d'"' -f2 | \
             tr 'A-Z' 'a-z')
ZCASHD="$ZCASHD_NAME $ZCASHD_VERSION"

echo "Connected to $ZEBRAD (port $ZEBRAD_RPC_PORT) and $ZCASHD (zcash.conf port)."

echo

ZEBRAD_BLOCKCHAIN_INFO="$ZCASH_RPC_TMP_DIR/$ZEBRAD_NAME-check-getblockchaininfo.json"
ZCASHD_BLOCKCHAIN_INFO="$ZCASH_RPC_TMP_DIR/$ZCASHD_NAME-check-getblockchaininfo.json"

echo "Checking $ZEBRAD network and tip height..."
$ZCASH_CLI $ZEBRAD_EXTRA_ARGS -rpcport="$ZEBRAD_RPC_PORT" getblockchaininfo > "$ZEBRAD_BLOCKCHAIN_INFO"

ZEBRAD_NET=$(cat "$ZEBRAD_BLOCKCHAIN_INFO" | grep '"chain"' | cut -d: -f2 | tr -d ' ,"')
ZEBRAD_HEIGHT=$(cat "$ZEBRAD_BLOCKCHAIN_INFO" | grep '"blocks"' | cut -d: -f2 | tr -d ' ,"')

echo "Checking $ZCASHD network and tip height..."
$ZCASH_CLI $ZCASHD_EXTRA_ARGS getblockchaininfo > "$ZCASHD_BLOCKCHAIN_INFO"

ZCASHD_NET=$(cat "$ZCASHD_BLOCKCHAIN_INFO" | grep '"chain"' | cut -d: -f2 | tr -d ' ,"')
ZCASHD_HEIGHT=$(cat "$ZCASHD_BLOCKCHAIN_INFO" | grep '"blocks"' | cut -d: -f2 | tr -d ' ,"')

echo

if [ "$ZEBRAD_NET" != "$ZCASHD_NET" ]; then
    echo "WARNING: comparing RPC responses from different networks:"
    echo "$ZEBRAD is on: $ZEBRAD_NET"
    echo "$ZCASHD is on: $ZCASHD_NET"
    echo
fi

if [ "$ZEBRAD_HEIGHT" -ne "$ZCASHD_HEIGHT" ]; then
    echo "WARNING: comparing RPC responses from different heights:"
    echo "$ZEBRAD is at: $ZEBRAD_HEIGHT"
    echo "$ZCASHD is at: $ZCASHD_HEIGHT"
    echo
fi

ZEBRAD_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZEBRAD_NAME-$ZEBRAD_NET-$ZEBRAD_HEIGHT-$1.json"
ZCASHD_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZCASHD_NAME-$ZCASHD_NET-$ZCASHD_HEIGHT-$1.json"

echo "Request:"
echo "$@"
echo

echo "Querying $ZEBRAD $ZEBRAD_NET chain at height >=$ZEBRAD_HEIGHT..."
time $ZCASH_CLI $ZEBRAD_EXTRA_ARGS -rpcport="$ZEBRAD_RPC_PORT" "$@" | jq -S > "$ZEBRAD_RESPONSE"
echo

echo "Querying $ZCASHD $ZCASHD_NET chain at height >=$ZCASHD_HEIGHT..."
time $ZCASH_CLI $ZCASHD_EXTRA_ARGS "$@" | jq -S > "$ZCASHD_RESPONSE"
echo

echo

echo "Response diff between $ZEBRAD and $ZCASHD (limited to ${OUTPUT_DATA_LINE_LIMIT} lines):"

RESPONSE_DIFF="$ZCASH_RPC_TMP_DIR/diff-$1.json"

$DIFF "$ZEBRAD_RESPONSE" "$ZCASHD_RESPONSE" > "$RESPONSE_DIFF"
EXIT_STATUS=$?
cat "$RESPONSE_DIFF" | head -$OUTPUT_DATA_LINE_LIMIT

if [[ "$EXIT_STATUS" -eq "0" ]]; then
    echo "RPC responses were identical"
    echo
    echo "$ZEBRAD_RESPONSE, limited to $OUTPUT_DATA_LINE_LIMIT lines:"
    export ZEBRAD_RESPONSE=$ZEBRAD_RESPONSE
    if [[ $(cat "$ZEBRAD_RESPONSE" | wc -l) -gt $OUTPUT_DATA_LINE_LIMIT ]]; then
        cat "$ZEBRAD_RESPONSE" | head -$((OUTPUT_DATA_LINE_LIMIT / 2)) || true
        echo "..."
        cat "$ZEBRAD_RESPONSE" | tail -$((OUTPUT_DATA_LINE_LIMIT / 2)) || true
    else
        cat "$ZEBRAD_RESPONSE"
    fi
fi

# Consistency checks between RPCs
#
# TODO:
# - sum of getaddressutxos.satoshis equals getaddressbalance
# - set of getaddressutxos.txid is a subset of getaddresstxids <addresses> 1 <max height>
# - getblockchaininfo.bestblockhash equals getbestblockhash

if [ "$1" == "getaddressutxos" ]; then
    set "getaddressbalance" "$2"
elif [ "$1" == "getrawmempool" ]; then
    # Call `getrawmempool` again as a dummy request (this script isn't set up to do multiple cross-check calls)
    set "getrawmempool"
else
    exit $EXIT_STATUS
fi

ZEBRAD_CHECK_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZEBRAD_NAME-$ZEBRAD_NET-$ZEBRAD_HEIGHT-$1.json"
ZCASHD_CHECK_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZCASHD_NAME-$ZCASHD_NET-$ZCASHD_HEIGHT-$1.json"

echo

echo "Cross-checking request:"
echo "$@"
echo

echo "Querying $ZEBRAD $ZEBRAD_NET chain at height >=$ZEBRAD_HEIGHT..."
$ZCASH_CLI $ZEBRAD_EXTRA_ARGS -rpcport="$ZEBRAD_RPC_PORT" "$@" > "$ZEBRAD_CHECK_RESPONSE"

echo "Querying $ZCASHD $ZCASHD_NET chain at height >=$ZCASHD_HEIGHT..."
$ZCASH_CLI $ZCASHD_EXTRA_ARGS "$@" > "$ZCASHD_CHECK_RESPONSE"

echo

echo "$1 diff between $ZEBRAD and $ZCASHD (limited to ${OUTPUT_DATA_LINE_LIMIT} lines):"

CHECK_DIFF="$ZCASH_RPC_TMP_DIR/diff-$1.json"

$DIFF "$ZEBRAD_CHECK_RESPONSE" "$ZCASHD_CHECK_RESPONSE" > "$CHECK_DIFF"
CHECK_EXIT_STATUS=$?
cat "$CHECK_DIFF" | head -$OUTPUT_DATA_LINE_LIMIT

if [[ "$CHECK_EXIT_STATUS" -eq "0" ]]; then
    echo "RPC check responses were identical"
    echo
    echo "$ZEBRAD_CHECK_RESPONSE, limited to $OUTPUT_DATA_LINE_LIMIT lines:"
    export ZEBRAD_CHECK_RESPONSE=$ZEBRAD_CHECK_RESPONSE
    if [[ $(cat "$ZEBRAD_CHECK_RESPONSE" | wc -l) -gt $OUTPUT_DATA_LINE_LIMIT ]]; then
        cat "$ZEBRAD_CHECK_RESPONSE" | head -$((OUTPUT_DATA_LINE_LIMIT / 2)) || true
        echo "..."
        cat "$ZEBRAD_CHECK_RESPONSE" | tail -$((OUTPUT_DATA_LINE_LIMIT / 2)) || true
    else
        cat "$ZEBRAD_CHECK_RESPONSE"
    fi
fi

if [ "$1" == "getaddressbalance" ]; then
    echo

    echo "Extracting getaddressbalance.balance..."

    ZEBRAD_NUM_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZEBRAD_NAME-$ZEBRAD_NET-$ZEBRAD_HEIGHT-getaddressbalance-num.txt"
    ZCASHD_NUM_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZCASHD_NAME-$ZCASHD_NET-$ZCASHD_HEIGHT-getaddressbalance-num.txt"

    cat "$ZEBRAD_CHECK_RESPONSE" | $JQ '.balance' > "$ZEBRAD_NUM_RESPONSE"
    cat "$ZCASHD_CHECK_RESPONSE" | $JQ '.balance' > "$ZCASHD_NUM_RESPONSE"

    echo "Summing getaddressutxos.satoshis..."

    ZEBRAD_SUM_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZEBRAD_NAME-$ZEBRAD_NET-$ZEBRAD_HEIGHT-getaddressutxos-sum.txt"
    ZCASHD_SUM_RESPONSE="$ZCASH_RPC_TMP_DIR/$ZCASHD_NAME-$ZCASHD_NET-$ZCASHD_HEIGHT-getaddressutxos-sum.txt"

    cat "$ZEBRAD_RESPONSE" | $JQ 'map(.satoshis) | add // 0' > "$ZEBRAD_SUM_RESPONSE"
    cat "$ZCASHD_RESPONSE" | $JQ 'map(.satoshis) | add // 0' > "$ZCASHD_SUM_RESPONSE"

    echo

    echo "Balance diff between $ZEBRAD and $ZCASHD:"
    echo "(for both getaddressbalance and getaddressutxos)"

    $DIFF --from-file="$ZEBRAD_NUM_RESPONSE" "$ZCASHD_NUM_RESPONSE" \
          "$ZEBRAD_SUM_RESPONSE" "$ZCASHD_SUM_RESPONSE" \
        && ( \
             echo "RPC balances were identical"; \
             echo ; \
             echo "$ZEBRAD_NUM_RESPONSE, limited to $OUTPUT_DATA_LINE_LIMIT lines:"; \
             export ZEBRAD_NUM_RESPONSE=$ZEBRAD_NUM_RESPONSE
             if [[ $(cat "$ZEBRAD_NUM_RESPONSE" | wc -l) -gt $OUTPUT_DATA_LINE_LIMIT ]]; then \
                 cat "$ZEBRAD_NUM_RESPONSE" | head -$((OUTPUT_DATA_LINE_LIMIT / 2)) || true; \
                 echo "..."; \
                 cat "$ZEBRAD_NUM_RESPONSE" | tail -$((OUTPUT_DATA_LINE_LIMIT / 2)) || true; \
             else \
                 cat "$ZEBRAD_NUM_RESPONSE"; \
             fi; \
        )

    COMPARE_EXIT_STATUS=$?

    if [ $COMPARE_EXIT_STATUS -ne 0 ]; then
        exit $COMPARE_EXIT_STATUS
    fi
fi

if [ "$1" == "getrawmempool" ] && [ $CHECK_EXIT_STATUS != 0 ]; then
    set TRANSACTION_ID
    set TRANSACTION_HEX_FILE
    set TRANSACTION_DECODED

    ZEBRAD_TRANSACTION_IDS=$(cat "$ZEBRAD_RESPONSE" | $JQ -r 'join("\n")' | head -$MEMPOOL_TX_LIMIT)
    ZCASHD_TRANSACTION_IDS=$(cat "$ZCASHD_RESPONSE" | $JQ -r 'join("\n")' | head -$MEMPOOL_TX_LIMIT)

    echo
    echo "# Dumping transactions from zebrad mempool (limited to ${MEMPOOL_TX_LIMIT})"
    echo

    for TRANSACTION_ID in $ZEBRAD_TRANSACTION_IDS; do
        TRANSACTION_HEX_FILE="$ZCASH_RPC_TMP_DIR/$ZEBRAD_NAME-$ZEBRAD_NET-$ZEBRAD_HEIGHT-$TRANSACTION_ID.json"

        $ZCASH_CLI $ZEBRAD_EXTRA_ARGS -rpcport="$ZEBRAD_RPC_PORT" getrawtransaction $TRANSACTION_ID 0 > $TRANSACTION_HEX_FILE

        echo "## Displaying transaction $TRANSACTION_ID from zebrad (limited to ${OUTPUT_DATA_LINE_LIMIT} lines)"
        echo

        # read the proposal data from a file, to avoid command-line length limits
        TRANSACTION_DECODED=`cat "$TRANSACTION_HEX_FILE" | \
            $ZCASH_CLI $ZCASHD_EXTRA_ARGS -stdin decoderawtransaction`

        echo $TRANSACTION_DECODED | $JQ | head -$OUTPUT_DATA_LINE_LIMIT
        echo
    done

    echo
    echo "# Dumping transactions from zcashd mempool  (limited to ${MEMPOOL_TX_LIMIT})"
    echo

    for TRANSACTION_ID in $ZCASHD_TRANSACTION_IDS; do
        TRANSACTION_HEX_FILE="$ZCASH_RPC_TMP_DIR/$ZCASHD_NAME-$ZCASHD_NET-$ZCASHD_HEIGHT-TRANSACTION_HEX-$TRANSACTION_ID.json"

        $ZCASH_CLI $ZCASHD_EXTRA_ARGS getrawtransaction $TRANSACTION_ID 0 > $TRANSACTION_HEX_FILE

        echo "## Displaying transaction $TRANSACTION_ID from zcashd (limited to ${OUTPUT_DATA_LINE_LIMIT} lines)"
        echo

        # read the proposal data from a file, to avoid command-line length limits
        TRANSACTION_DECODED=`cat "$TRANSACTION_HEX_FILE" | \
            $ZCASH_CLI $ZCASHD_EXTRA_ARGS -stdin decoderawtransaction`

        echo $TRANSACTION_DECODED | $JQ | head -$OUTPUT_DATA_LINE_LIMIT
        echo
    done

fi

echo "Full RPC output is in $ZCASH_RPC_TMP_DIR"

if [ $EXIT_STATUS -ne 0 ]; then
    exit $EXIT_STATUS
else
    exit $CHECK_EXIT_STATUS
fi