Provisioning Bitcoin Network Crawler

Addy Yeow edited this page on Aug 7, 2017 · 52 revisions

Launch Server

Hetzner PX91-SSD Dedicated Server 128GB Debian 8 64-bit (Nuremberg, DE)

Initial Login

ssh root@[IP_ADDRESS]
    [PASSWORD]

Change Password

passwd
    [PASSWORD]

Configure Hostname

vi /etc/hostname
    [HOSTNAME]

vi /etc/hosts
    127.0.0.1 localhost
    [IP_ADDRESS] [HOSTNAME]
    ::1 ip6-localhost ip6-loopback
    fe00::0 ip6-localnet
    ff00::0 ip6-mcastprefix
    ff02::1 ip6-allnodes
    ff02::2 ip6-allrouters
    ff02::3 ip6-allhosts
    [IPV6_ADDRESS] [HOSTNAME]

Configure Locales

dpkg-reconfigure locales
    en_US.UTF-8 UTF-8
    en_US.UTF-8

vi /etc/environment
    LC_CTYPE=en_US.UTF-8

Configure Timezone

dpkg-reconfigure tzdata
    None of the above
    UTC

Update Packages

apt-get update; apt-get upgrade

Install Packages

apt-get -y install htop sudo unzip build-essential python-dev python-virtualenv tcl subversion

Add User

adduser [USERNAME]
    [PASSWORD]
adduser [USERNAME] sudo
mkdir -p /home/[USERNAME]/.ssh; chmod 700 /home/[USERNAME]/.ssh
vi /home/[USERNAME]/.ssh/authorized_keys
    [PUBLICKEY]
chmod 600 /home/[USERNAME]/.ssh/authorized_keys; chown -R [USERNAME]:[USERNAME] /home/[USERNAME]/.ssh

Update /home/[USERNAME]/.bashrc

export REDIS_SOCKET=/tmp/redis.sock
export REDIS_PASSWORD=[PASSWORD]

Update /etc/ssh/sshd_config

Port [PORT]
LoginGraceTime 60
PermitRootLogin no
AllowUsers [USERNAME]
PasswordAuthentication no

Update /etc/sysctl.conf

net.ipv4.conf.default.rp_filter=1
net.ipv4.conf.all.rp_filter=1
net.ipv4.tcp_syncookies=1
net.ipv4.conf.all.accept_redirects=0
net.ipv6.conf.all.accept_redirects=0
net.ipv4.conf.all.send_redirects=0
net.ipv4.conf.all.accept_source_route=0
net.ipv6.conf.all.accept_source_route=0
net.ipv4.conf.all.log_martians=1
net.core.rmem_default=33554432
net.core.wmem_default=33554432
net.core.rmem_max=33554432
net.core.wmem_max=33554432
net.core.optmem_max=33554432
net.ipv4.tcp_rmem=10240 87380 33554432
net.ipv4.tcp_wmem=10240 87380 33554432
net.ipv4.ip_local_port_range=2000 65500
net.core.netdev_max_backlog=100000
net.ipv4.tcp_max_syn_backlog=80000
net.ipv4.tcp_max_tw_buckets=2000000
net.ipv4.tcp_tw_reuse=1
net.ipv4.tcp_fin_timeout=5
net.ipv4.tcp_slow_start_after_idle=0
net.core.somaxconn=60000
fs.file-max=1000000
vm.swappiness=10
vm.min_free_kbytes=1048576
vm.overcommit_memory=1

Update /etc/security/limits.conf

* soft nofile 1000000
* hard nofile 1000000

Update /etc/rc.local

echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
/sbin/ifconfig eth0 txqueuelen 5000

Reboot Server

reboot

Normal User Login

ssh -p [PORT] -i [PRIVATEKEY] [USERNAME]@[IP_ADDRESS]

Install Redis

cd; wget http://download.redis.io/releases/redis-3.2.8.tar.gz
tar xzf redis-3.2.8.tar.gz; cd redis-3.2.8; make; make test
sudo make install
cd utils; sudo bash install_server.sh
sudo vi /etc/init.d/redis_0
*******************************************************************************
#!/bin/sh

##### BEGIN INIT INFO
# Provides:          redis_0
# Required-Start:    $network $local_fs $remote_fs
# Required-Stop:     $network $local_fs $remote_fs
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Should-Start:      $syslog $named
# Should-Stop:       $syslog $named
# Short-Description: Starts redis_0
# Description:       Starts redis_0 using start-stop-daemon
##### END INIT INFO

EXEC=/usr/local/bin/redis-server
CLIEXEC=/usr/local/bin/redis-cli
PIDFILE=/var/run/redis_0.pid
CONF="/etc/redis/0.conf"
REDISSOCKET="/tmp/redis.sock"
REDISPASSWORD="[PASSWORD]"

case "$1" in
    start)
        if [ -f $PIDFILE ]
        then
            echo "$PIDFILE exists, process is already running or crashed"
        else
            echo "Starting Redis server..."
            $EXEC $CONF
        fi
        ;;
    stop)
        if [ ! -f $PIDFILE ]
        then
            echo "$PIDFILE does not exist, process is not running"
        else
            PID=$(cat $PIDFILE)
            echo "Stopping ..."
            $CLIEXEC -s $REDISSOCKET -a $REDISPASSWORD shutdown
            while [ -x /proc/${PID} ]
            do
                echo "Waiting for Redis to shutdown ..."
                sleep 1
            done
            echo "Redis stopped"
        fi
        ;;
    *)
        echo "Please use start or stop as first argument"
        ;;
esac
*******************************************************************************
sudo update-rc.d redis_0 defaults
sudo vi /etc/redis/0.conf
    unixsocket /tmp/redis.sock
    unixsocketperm 777
    #save 900 1
    #save 300 10
    #save 60 10000
    requirepass [PASSWORD]
    maxclients 50000
    maxmemory 68719476736
    maxmemory-policy volatile-ttl
    notify-keyspace-events K$z
    activerehashing no
    client-output-buffer-limit slave 512mb 256mb 300
    client-output-buffer-limit pubsub 512mb 256mb 300
    hz 20

Reboot Server

sudo reboot

Normal User Login

ssh -p [PORT] -i [PRIVATEKEY] [USERNAME]@[IP_ADDRESS]

Launch Crawler

cd; wget --no-check-certificate https://github.com/ayeowch/bitnodes/archive/master.zip
unzip master.zip
virtualenv ~/.virtualenvs/bitnodes
source ~/.virtualenvs/bitnodes/bin/activate
cd bitnodes-master
pip install -r requirements.txt
bash geoip/update.sh
bash start.sh
cd data/pcap/f9beb4d9
sudo rm *.pcap; sudo tcpdump -i eth0 -w %s.eth0.pcap -v -n -G 2 -B 65536 -Z [USERNAME] 'tcp and not src host [IP_ADDRESS] and not src host [IPV6_ADDRESS]' > eth0 2>&1 &
sudo tcpdump -i lo -w %s.lo.pcap -v -n -G 2 -B 65536 -Z [USERNAME] 'tcp and port 9050' > lo 2>&1 &