tor-netdoc 0.45.0

#!/usr/bin/env bash
#
# usage:
#   testdata-live-download
#
# This has to be run inside the git repository!
#
#
# This script can also be used to help test with a *complete* consensus or vote:
# Run it, and then
#     TOR_NETDOC_TESTDATA_LIVE_PREFIX=$PWD/crates/tor-netdoc/testdata-live/.entire. cargo test -p tor-netdoc --all-features roundtrip_live
#
# The actual downloaded files are .gitignored and should not be directly committed.
# This script will download each file with `curl` only once, not re-downloading files
# it already has.  It will copy and/or process these to non-ignored output files -
# see list below.  *Those* should be committed.
#
#
# There is currently no facility to regenerate the output without also downloading new data.
# That's nontrivial because we don't commit the whole consensus (and don't want to).
#
#
# Output, all in crates/tor-netdoc/testdata-live:
#
#     authority                         Vote
#     consensus                         Consensus (plain)
#     consensus-microdesc               Consensus (md)
#
#     NETSTATUS--entry--NICKNAME        Routerstatus entry for NICKNAME in NETSTATUS
#     NETSTATUS--desc--NICKNAME         (Micro)descriptor referenced from NETSTATUS

set -euo pipefail

case "$#" in
0) ;;
*) echo >&2 'bad usage'; exit 8 ;;
esac

root="$(git rev-parse --show-toplevel)"
out="$root/crates/tor-netdoc/testdata-live"
cd "$out"

# We include only a handful of relays, because:
#  - routerdescs can contain personal data that we ought not to immemorialise in git
#  - we don't want a whole consensus because it's too large

files=(consensus authority consensus-microdesc)

declare -A rust_per_relay_data

run-curl () {
    local url="$1"
    local file="$2"
    if test -s "$file"; then return; fi
    echo >&2 "fetching $url"
    curl -f "http://$url" >"$file.tmp"
    mv -f "$file"{.tmp,}
}

rust_per_variety () {
    local suffix="$1"
    printf "PerVariety {
"
    for f in "${files[@]}"; do
	printf '            '
	case $f in
	    consensus)           printf "plain: " ;;
	    consensus-microdesc) printf "md:    " ;;
	    authority)           printf "vote:  " ;;
	    *) echo >&2 "unhandled for rust_per_variety $f"; exit 16 ;;
	esac
	printf 'include_str!("%s"),
' "$f$suffix"
    done

    printf "        }"
}

rust_per_relay () {
    local suffix=$1
    printf '    PerRelay {
        nick: "%s",
        data: ' "$nick"
    rust_per_variety "--$suffix--$nick"
    printf "
    },"
}

start () {
    for f in "${files[@]}"; do
        run-curl "$auth/tor/status-vote/current/$f" .entire."$f"

	perl -ne 'print or die $! unless m/^r /..0' \
	     <.entire."$f" >"$f.new"
    done
    rust_module=\
"//! Automatically generated by testdata-live-download - DO NOT EDIT

use super::*;

/// Network status documents
pub const NETSTATUS: PerVariety =
        $(rust_per_variety "");
"
}

include_relay () {
    id=$1; nick=$2

    for f in "${files[@]}"; do
	ent_file="$f--entry--$nick"
	perl -ne '
              BEGIN { $exp = shift @ARGV; }
    	      $y = ($1 eq $exp) if m/^r (\S+ \S+) /;
	      print or die $! if $y;
        ' <.entire."$f" >"$ent_file" "$nick $id"
	cat "$ent_file">>"$f.new" 

	case "$f" in
	    consensus|authority)
		digest=$(perl <"$ent_file" -ne '
		    use MIME::Base64;
		    next unless m{^r \S+ \S+ (\S+) };
		    print uc unpack "H*", decode_base64($1);
		')
		url="$auth/tor/server/d"
		;;
	    consensus-microdesc)
		digest=$(perl <"$ent_file" -ne '
                    next unless m{^m ([+/0-9a-zA-Z]+)\s};
                    print $1;
                ')
		url="$auth/tor/micro/d"
		;;
	    *)
		echo >&2 'Unknown file type!'
		exit 1
		;;
	esac

	# Use a temporary filename based on the digest, so that run-curl won't reuse
	# a previously-downloaded file with a different digest.
	desc_file_tmp=.tmp."$f--desc--$nick.${digest//\//_}"
        run-curl "$url/$digest" "$desc_file_tmp"
        cp "$desc_file_tmp" "$f--desc--$nick"

	rust_per_relay_data[ROUTERSTATUSES]+="$(rust_per_relay entry)
"
	rust_per_relay_data[DESCRIPTORS]+="$(rust_per_relay desc)
"
    done
}

rust_relay_list () {
    local var=$1; shift
    # stdin is here document with documentation in Rust syntax

    rust_module+=\
"
$(cat)
pub const RELAY_$var: &[PerRelay] = &[
${rust_per_relay_data[$var]}];
"
}

finish () {
    for f in "${files[@]}"; do
	perl -ne 'print or die $! if m/^directory-footer\s/..0' \
	     <.entire."$f" >>"$f.new"

	mv "$f"{.new,}
    done

    rust_relay_list ROUTERSTATUSES <<'END'
/// Relays' routerstatus entries (excerpts from each corresponding network status)
END
    rust_relay_list DESCRIPTORS <<'END'
/// Relays' router descriptors and microdescriptors
END
    printf "%s" "$rust_module" >generated_consts.rs

    echo
    echo "Updated $out.  Don't forget to git add."
}

#---------- configuration section ----------

# which dirauth we use
#auth=faravahar.redteam.net
auth=tor.cypherpunks.eu

start

# which relays we include
# must be sorted by id
include_relay 'AAAErLudKby6FyVrs1ko3b/Iq6k' lisdex
include_relay 'GiXGNY25E0KqUXIKUDi3J0JzJJg' moria1
include_relay 'MDUJq5EO8ge3Q4wnQ1xKL9V58bE' Ukko
include_relay 'UEheA8o505O9VNMVzrpl5t0P3bk' d2d4
include_relay 'VpJ+YbUebzY/tVSYFQpt389wd/I' Akka

finish