bondingadmin-install/migrate

#!/bin/bash -e
#
# migrate - Migrate bondingadmin from one server to another
#

VARS=/var/lib/bondingadmin-migrate/

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"

# function_defined <name>
#
# True if the function is defined
#
function function_defined() {
    declare -F | grep -qe "\b$1\b"
}

function check_args() {
    for arg_def in "$@" ; do
        OLD_IFS="$IFS"
        IFS=","
        set $arg_def
        IFS="$OLD_IFS"
        name="$1"
        skip="$2"
        regex="$3"
        value="$4"
        if [ -z $skip ] ; then
            if [ -z $value ] ; then
                echo "Argument '$name' is required" >&2
                return 1
            fi
        fi
        if [ ! -z $regex ] ; then
            if [ ! -z $value ] ; then
                if ! [[ $value =~ $regex ]] ; then
                    echo "Value '$value' for argument '$name' does not match pattern $regex" >&2
                    return 1
                fi
            fi
        fi
        echo "$name=$value"
    done
}

# get_var <var> [default]
#
# Get variable from data store. If unset, return default
#
function get_var() {
    varfile="${VARS}/$1"
    if [ -f "$varfile" ] ; then
        cat $varfile
    else
        echo "$2"
    fi
}


# Set variable in data store
#
function set_var() {
    varfile="${VARS}/$1"
    vardir=$(dirname $varfile)
    if [ ! -d $vardir ] ; then
        install -d -m 0755 $vardir
    fi
    echo -n "$2" > $varfile
}


# Delete variable from data store
function del_var() {
    varfile="${VARS}/$1"
    vardir=$(dirname $varfile)
    rm -f $varfile

    # Clean up empty dirs
    find $VARS -depth -type d -empty -exec rmdir {} \;
}


# Check if variable exists
#
function has_var() {
    test -f ${VARS}/$1
}


# List variables in data store. If an argument is specified, list only
# variables under that root.
#
function all_vars() {
    for var in $(find $VARS/$1 -type f | sort) ; do
        echo ${var#$VARS}
    done
}


# Dump variables in data store. If an argument is specified, list only
# variables under that root.
#
function dump_vars() {
    for var in $(find $VARS/$1 -type f | sort) ; do
        echo "${var#$VARS} $(cat $var)"
    done
}


# List immediate variables in data store. If an argument is specified, list
# only variables under that root.
#
function list_vars() {
    if [ -d "$VARS/$1" ] ; then
        for var in $(find $VARS/$1 -mindepth 1 -maxdepth 1 -type f | sort) ; do
            echo ${var#$VARS}
        done
    fi
}

# ask_value <msg> <validator>
#
# Get value from user. The value will not be returned until the validator is
# successful
#
function ask_value() {
    msg=$1
    validator=$2

    valid=false
    while test "$valid" = "false" ; do
        echo -n "$msg " >&2
        read value
        $validator $value && valid=true || valid=false
    done
    echo $value
}

function validate_yn() {
    if [ "$1" = "y" -o "$1" = "n" ] ; then
        return 0
    fi

    return 1
}


function validate_ip() {
    ip r get $1 > /dev/null
}

function get_bondingadmin_option() {
    name=$1
    python3 -c "import configparser; c=configparser.ConfigParser(); c.read('/etc/bondingadmin/bondingadmin.conf'); print(c['partner']['$name'])"
}

function source_failed() {
    echo "Source check failed. Please fix the issue and try again"
}

function check_source() {
    ret=0

    trap source_failed ERR

    # Get server hostname
    #
    if ! has_var hostname ; then
        hostname=$(grep -e '^mgmt_server_url' /etc/bondingadmin/bondingadmin.conf | cut -d '=' -f 2)
        set_var hostname $hostname
    fi

    # Check if we want to sync influx
    #
    if ! has_var sync_influx ; then
        sync_influx=$(ask_value "Sync influx data? (y/n)" validate_yn)
        set_var sync_influx $sync_influx
    fi

    # Get IP of target server
    #
    if ! has_var target_ip ; then
        target_ip=$(ask_value "Enter the target server IP:" validate_ip)
        set_var target_ip $target_ip
    fi

    # Make sure the target host can access this one
    #
    if which bondingadmin-nftables >/dev/null ; then
        nft_file=/etc/bondingadmin/nftables/filter-input-bondingadmin-migrate.nft
        if [ ! -f $nft_file ] ; then
            echo "ip saddr $(get_var target_ip) accept" > $nft_file
            bondingadmin-nftables start
        fi
    else
        known_ips=/etc/firewall.d/known_ips
        target_ip_present=false
        target_ip=$(get_var target_ip)
        if [ -f $known_ips ] ; then
            if grep -q $target_ip $known_ips ; then
                target_ip_present=true
            fi
        fi
        if [ $target_ip_present = "false" ] ; then
            echo "iptables -A \$CHAIN -s $target_ip -j ACCEPT" >> $known_ips
            systemctl restart firewall
        fi
    fi

    # Check TTL of hostname
    #
    if ! has_var ttl_complete ; then
        which dig >/dev/null || apt install -y dnsutils
        hostname=$(get_var hostname)
        ttl=$(dig +nocmd +noall +answer $hostname | awk '{print $2}')
        if [ -z $ttl ] ; then
            echo "Could not get TTL of $hostname"
            return 1
        fi
        if [ "$ttl" -le 300 ] ; then
            set_var ttl_complete 1
        else
            echo "TTL of $hostname is $ttl. Please set it to 300 or less before continuing"
            ret=1
        fi
    fi

    # Check for hostname in node hosts files
    #
    if ! has_var hosts_not_overidden ; then
        hostname=$(get_var hostname)
        tmpfile=$(mktemp)
        salt --hide-timeout --out=txt '*' cmd.run "grep $hostname /etc/hosts ||:" >$tmpfile 2>/dev/null
        if grep -q $hostname $tmpfile ; then
            echo "At least one node has the hostname overridden in /etc/hosts. Please fix before continuing:"
            echo
            cat $tmpfile
            ret=1
        else
            set_var hosts_not_overidden 1
        fi
        rm $tmpfile
    fi

    # Check for wrong server in bonding.conf
    #
    if ! has_var bonding_conf_server_correct ; then
        hostname=$(get_var hostname)
        tmpfile=$(mktemp)
        salt --hide-timeout --out=txt '*' cmd.run "grep -e '^server' /etc/bonding/bonding.conf | grep -v $hostname ||:" >$tmpfile 2>/dev/null
        if grep -q server $tmpfile ; then
            echo "At least one node has the wrong server hostname in /etc/bonding/bonding.conf. Please fix before continuing:"
            echo
            cat $tmpfile
            ret=1
        else
            set_var bonding_conf_server_correct 1
        fi
        rm $tmpfile
    fi

    # Check for management server ping, but only if there are no hosts entries
    #
    if has_var hosts_not_overidden ; then
        if ! has_var nodes_reach_management ; then
            hostname=$(get_var hostname)
            tmpfile=$(mktemp)
            salt --hide-timeout --out=txt '*' cmd.run "ping -c 1 $hostname ||:" >$tmpfile 2>/dev/null
            if grep -q "100% packet loss" $tmpfile ; then
                echo "At least one node cannot ping the management server by name. Please fix before continuing:"
                echo
                grep "100% packet loss" $tmpfile | cut -d ':' -f 1
                ret=1
            else
                set_var nodes_reach_management 1
            fi
            rm $tmpfile
        fi
    fi

    # Check for software up-to-date
    #
    if ! has_var bondingadmin_up_to_date ; then
        apt-get update --allow-releaseinfo-change >/dev/null
        if apt list --upgradable | grep -q bondingadmin ; then
            echo "Bondingadmin is not up-to-date. Please upgrade before continuing"
            ret=1
        else
            set_var bondingadmin_up_to_date 1
        fi
    fi

    # Get API token
    #
    if ! has_var multapplied_api_token ; then
        set_var multapplied_api_token $(ba get multapplied_api_token)
    fi

    # Get database locale
    #
    if ! has_var db_locale ; then
        db_locale=$(su postgres -c "psql -Atc \"select datcollate from pg_catalog.pg_database where datname = 'bondingadmin';\"")
        set_var db_locale $db_locale
    fi

    if [ $ret = 0 ] ; then
        release=$(echo "import version; print(version.__version__)" | PYTHONPATH="/usr/lib/bondingadmin/" python3 -c "import version; print(version.__version__)")
        echo
        echo "Pre-migration checks completed. To continue, run the following on the target"
        echo "server in a tmux session:"
        echo
        echo "    curl 'https://git.multapplied.net/Partner/bondingadmin-install/archive/master.tar.gz' | tar -xz"
        echo "    cd bondingadmin-install"
        echo "    ./migrate target-pre $(get_var hostname)"
        echo
    fi

    # Get config entries for the target install
    #
    for name in full_name short_name email country province city ; do
        set_var $name "get_bondingadmin_option $name"
    done


    return $ret
}

function check_target() {
    ret=0

    if ! has_var bondingadmin_installed ; then
        if dpkg -l bondingadmin | grep -q bondingadmin ; then
            set_var bondingadmin_installed 1
        else
            echo "Bondingadmin is not installed"
            ret=1
        fi
    fi

    if ! has_var bondingadmin_up_to_date ; then
        apt-get update >/dev/null
        if apt list --upgradable | grep -q bondingadmin ; then
            echo "Bondingadmin is not up-to-date. Please upgrade before continuing"
            ret=1
        else
            set_var bondingadmin_up_to_date 1
        fi
    fi

    # Check that locale matches the source
    #
    locale=$(su postgres -c "psql -Atc \"select datcollate from pg_catalog.pg_database where datname = 'bondingadmin';\"")
    source_locale=$(get_var source_locale)
    if [ "$locale" != "$source_locale" ] ; then
        echo "The database locale does not match the source server. You will need to"
        echo "recreate with the matching locale."
        echo
        answer=$(ask_value "Do you want to recreate now? (y/n)" validate_yn)
        if [ "$answer" = "y" ] ; then
            echo "$source_locale" > /etc/locale.gen
            locale-gen
            update-locale LANG="$source_locale" LANGUAGE="$source_locale"
            systemctl stop postgresql
            rm -rf /var/lib/postgresql/* /etc/postgresql/*
            apt-get install -y postgresql --reinstall
            pg_createcluster --locale "$source_locale" --start 11 main
            systemctl start postgresql
            /usr/share/bondingadmin/initdb.sh
            echo "Database recreated"
        else
            echo "Please recreate the database and run this command again"
            return 1
        fi
    fi

    return $ret
}


function sync_influx() {
    if [ "$(get_var sync_influx)" = "y" ] ; then
        echo "Syncing influxdb data"
        rsync -a $(get_var hostname):/var/lib/influxdb/ /var/lib/influxdb/
    else
        echo "Setting up influxdb"
        /usr/lib/bondingadmin/influxconfig
    fi
}


usage_source="Run pre-migration actions on the source host"
function action_source() {
    check_source
}


usage_target_pre="<hostname>  Run pre-migration actions on the target host"
function action_target_pre() {
    hostname=$1

    if [ -z $hostname ] ; then
        echo "ERROR: No host name given"
        return 1
    fi

    # Check to see if we can access the target host
    #
    if [ -z $SSH_AUTH_SOCK ] ; then
        echo "No SSH agent detected. Please log back in with agent forwarding"
        return 1
    fi

    if [ -z $TMUX ] ; then
        echo "Please run this in a tmux session"
        return 1
    fi

    # Sync the datastore from the source host
    #
    rsync -a $hostname:/var/lib/bondingadmin-migrate/ /var/lib/bondingadmin-migrate/

    # Make sure the hostname matches
    #
    hostnamectl set-hostname $(get_var hostname)

    # Set the source IP
    #
    set_var source_ip $(python3 -c "import socket; print(socket.gethostbyname('$hostname'))")

    if ! has_var target_installed ; then
        cat <<EOF | debconf-set-selections
bondingadmin bondingadmin/mgmt_server_url string $(get_var hostname)
bondingadmin bondingadmin/full_name string $(get_var full_name)
bondingadmin bondingadmin/short_name string $(get_var short_name)
bondingadmin bondingadmin/email string $(get_var email)
bondingadmin bondingadmin/country string $(get_var country)
bondingadmin bondingadmin/province string $(get_var province)
bondingadmin bondingadmin/city string $(get_var city)
EOF

        $SCRIPT_DIR/install.sh --migration
        set_var target_installed 1
    fi

    check_target

    sync_influx

    set_var target_pre_complete

    echo
    echo "Pre-migration completed. When it's time to perform the actual migration, run"
    echo "the following command:"
    echo
    echo "    ./migrate target"
    echo
}

usage_target="Run migration actions on the target host"
function action_target() {
    if ! has_var target_pre_complete ; then
        echo "Pre-migration not complete. Please run the following command:"
        echo
        echo "    ./migrate target-pre <hostname>"
        echo
        return 1
    fi

    if [ -z $TMUX ] ; then
        echo "Please run this in a tmux session"
        return 1
    fi

    source_ip=$(get_var source_ip)

    # Stop the services on the new host
    systemctl stop bondingadmin

    # Remove the CA directory on the new host if it exists
    rm -rf /var/lib/bondingadmin/ca

    # Stop the services on the old host
    echo -e "Stopping bondingadmin bondingadmin-salt-minion and salt-master on old host"
    ssh "root@${source_ip}" "systemctl stop bondingadmin bondingadmin-salt-minion bondingadmin-salt-master mgmtvpn"

    # Run backup on old host
    echo -e "Running backup-bondingadmin on old host"
    ssh "root@${source_ip}" "backup-bondingadmin"

    # Copy backups from old server to new server
    echo -e "Copying backups from old host to new host"
    currentdate=$(date +%Y-%m-%d)
    mkdir -p "/var/lib/bondingadmin/backups/"
    scp "root@${source_ip}:/var/lib/bondingadmin/backups/*.${currentdate}.*" "/var/lib/bondingadmin/backups/"

    # Restore backup on new host
    echo -e "Running restore-bondingadmin on new (current) host"
    restore-bondingadmin

    # Enable all services except aggfail
    echo -ne "Enable all services except aggfail"
    systemctl enable bondingadmin bondingadmin-uwsgi homestead
    systemctl start bondingadmin
    systemctl stop aggfail

    set_var target_complete

    echo
    echo "Main migration completed. Next steps are:"
    echo
    echo "  1. Update the A/AAAA DNS records as appropriate"
    echo "  2. Check HTTPS TLS certificate"
    echo "  2. Wait for nodes to connect to the management server"
    echo "  3. Ensure nodes are accessible via salt:"
    echo
    echo "    salt '*' test.ping"
    echo
    echo "Once complete continue with the post-migration using the following command:"
    echo
    echo "    ./migrate target-post"
    echo
}


usage_target_post="Run post-migration actions on the target host"
function action_target_post() {
    if ! has_var target_complete ; then
        echo "Main migration not complete. Please run the following command:"
        echo
        echo "    ./migrate target"
        echo
        return 1
    fi

    if [ -z $TMUX ] ; then
        echo "Please run this in a tmux session"
        return 1
    fi

    source_ip=$(get_var source_ip)

    echo -e "Enabling aggfail service"
    systemctl enable --now aggfail

    echo -e "Syncing bonding repositories"
    ba sync_bonding_release

    echo -e "Regenerating bonding ISOs"
    ba regenerate_isos

    echo -e "Running backup-bondingadmin"
    backup-bondingadmin

    echo -e "Running sendmgmtpublickey"
    ba sendmgmtpublickey

    echo -e "Running remotebackup"
    ba remotebackup

    echo -e "Restarting firewall service"
    systemctl restart firewall

    # Migrate influxDB
    echo -ne "Migrating influxDB "

    if [ "$(get_var sync_influx)" = "y" ] ; then
        # stop influxdb on old host
        echo -e "Stopping influxdb on old host"
        ssh "root@${source_ip}" "systemctl stop influxdb"

        # stop influxdb on new host
        echo -e "Stopping influxdb on current host"
        systemctl stop influxdb

        # rsync influxdb from old to new host
        echo -e "rsyncing influxdb from old to new host"
        rsync -a "root@${source_ip}:/var/lib/influxdb/" "/var/lib/influxdb"

        # start influxdb on new host
        echo -e "Starting influxdb on current host"
        systemctl start influxdb
    fi
}


function get_actions() {
    for fn in $(declare -F | grep -e '\baction_' | cut -d ' ' -f 3) ; do
        action=${fn#action_}
        echo ${action//_/-}
    done
}

usage_actions="  Get a list of defined actions"
function action_actions() {
    echo $(get_actions)
}


usage_help="  Get help for action"
function action_help() {
    if [ -z "$1" ] ; then
        echo "No action specified"
        return 1
    fi
    action=${1//-/_}
    action_function="action_$action"
    if ! function_defined $action_function ; then
        echo "$1 is not an action"
        return 1
    fi
    usage_var="usage_$action"
    echo "$1 ${!usage_var}"
    help_function="help_$action"
    if function_defined $help_function ; then
        echo
        $help_function
    fi
}


usage_set="<option> <value>  Set config option to value"
function action_set() {
    args=$(check_args option,,,$1 value,,,$2) || return 1
    eval $args

    set_var $option $value
}


usage_get="<option> <value>  Get config option"
function action_get() {
    args=$(check_args option,,,$1) || return 1
    eval $args

    get_var $option
}


usage_dump="[base]  Dump config parameters"
function action_dump() {
    dump_vars $1
}


# Get script actions
actions=$(get_actions)

function usage() {
    echo -n "Usage: $0 [-x] <"
        sep=""
    for action in $actions ; do
        echo -n "${sep}${action}"
        sep="|"
    done
    echo "> [options]"
    echo
    echo "Actions:"
    for action in $actions ; do
        usage_var="usage_$action"
        usage_var=${usage_var//-/_}
        echo "    $action ${!usage_var}"
    done
}

if [ "$1" = "-x" ] ; then
    set -x
    shift
fi

ACTION="$1"

if [ -z $ACTION ] ; then
    usage
    echo "Action required"
    exit 0
fi

shift

action_function="action_${ACTION//-/_}"
if function_defined $action_function ; then
    $action_function "$@"
else
    echo "Unknown action $ACTION"
    exit 1
fi