#!/bin/bash -e # # migrate - Migrate bondingadmin from one server to another # VARS=/var/lib/bondingadmin-migrate/ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # function_defined # # True if the function is defined # function function_defined() { declare -F | grep -qe "\b$1\b" } function check_args() { for arg_def in "$@" ; do OLD_IFS="$IFS" IFS="," set $arg_def IFS="$OLD_IFS" name="$1" skip="$2" regex="$3" value="$4" if [ -z $skip ] ; then if [ -z $value ] ; then echo "Argument '$name' is required" >&2 return 1 fi fi if [ ! -z $regex ] ; then if [ ! -z $value ] ; then if ! [[ $value =~ $regex ]] ; then echo "Value '$value' for argument '$name' does not match pattern $regex" >&2 return 1 fi fi fi echo "$name=$value" done } # get_var [default] # # Get variable from data store. If unset, return default # function get_var() { varfile="${VARS}/$1" if [ -f "$varfile" ] ; then cat $varfile else echo "$2" fi } # Set variable in data store # function set_var() { varfile="${VARS}/$1" vardir=$(dirname $varfile) if [ ! -d $vardir ] ; then install -d -m 0755 $vardir fi echo -n "$2" > $varfile } # Delete variable from data store function del_var() { varfile="${VARS}/$1" vardir=$(dirname $varfile) rm -f $varfile # Clean up empty dirs find $VARS -depth -type d -empty -exec rmdir {} \; } # Check if variable exists # function has_var() { test -f ${VARS}/$1 } # List variables in data store. If an argument is specified, list only # variables under that root. # function all_vars() { for var in $(find $VARS/$1 -type f | sort) ; do echo ${var#$VARS} done } # Dump variables in data store. If an argument is specified, list only # variables under that root. # function dump_vars() { for var in $(find $VARS/$1 -type f | sort) ; do echo "${var#$VARS} $(cat $var)" done } # List immediate variables in data store. If an argument is specified, list # only variables under that root. # function list_vars() { if [ -d "$VARS/$1" ] ; then for var in $(find $VARS/$1 -mindepth 1 -maxdepth 1 -type f | sort) ; do echo ${var#$VARS} done fi } # ask_value # # Get value from user. The value will not be returned until the validator is # successful # function ask_value() { msg=$1 validator=$2 valid=false while test "$valid" = "false" ; do echo -n "$msg " >&2 read value $validator $value && valid=true || valid=false done echo $value } function validate_yn() { if [ "$1" = "y" -o "$1" = "n" ] ; then return 0 fi return 1 } function validate_ip() { ip r get $1 > /dev/null } function get_bondingadmin_option() { name=$1 python3 -c "import configparser; c=configparser.ConfigParser(); c.read('/etc/bondingadmin/bondingadmin.conf'); print(c['partner']['$name'])" } function source_failed() { echo "Source check failed. Please fix the issue and try again" } function check_source() { ret=0 trap source_failed ERR # Get server hostname # if ! has_var hostname ; then hostname=$(grep -e '^mgmt_server_url' /etc/bondingadmin/bondingadmin.conf | cut -d '=' -f 2) set_var hostname $hostname fi # Check if we want to sync influx # if ! has_var sync_influx ; then sync_influx=$(ask_value "Sync influx data? (y/n)" validate_yn) set_var sync_influx $sync_influx fi # Get IP of target server # if ! has_var target_ip ; then target_ip=$(ask_value "Enter the target server IP:" validate_ip) set_var target_ip $target_ip fi # Make sure the target host can access this one # if which bondingadmin-nftables >/dev/null ; then nft_file=/etc/bondingadmin/nftables/filter-input-bondingadmin-migrate.nft if [ ! -f $nft_file ] ; then echo "ip saddr $(get_var target_ip) accept" > $nft_file bondingadmin-nftables start fi else known_ips=/etc/firewall.d/known_ips target_ip_present=false target_ip=$(get_var target_ip) if [ -f $known_ips ] ; then if grep -q $target_ip $known_ips ; then target_ip_present=true fi fi if [ $target_ip_present = "false" ] ; then echo "iptables -A \$CHAIN -s $target_ip -j ACCEPT" >> $known_ips systemctl restart firewall fi fi # Check TTL of hostname # if ! has_var ttl_complete ; then which dig >/dev/null || apt install -y dnsutils hostname=$(get_var hostname) ttl=$(dig +nocmd +noall +answer $hostname | awk '{print $2}') if [ -z $ttl ] ; then echo "Could not get TTL of $hostname" return 1 fi if [ "$ttl" -le 300 ] ; then set_var ttl_complete 1 else echo "TTL of $hostname is $ttl. Please set it to 300 or less before continuing" ret=1 fi fi # Check for hostname in node hosts files # if ! has_var hosts_not_overidden ; then hostname=$(get_var hostname) tmpfile=$(mktemp) salt --hide-timeout --out=txt '*' cmd.run "grep $hostname /etc/hosts ||:" >$tmpfile 2>/dev/null if grep -q $hostname $tmpfile ; then echo "At least one node has the hostname overridden in /etc/hosts. Please fix before continuing:" echo cat $tmpfile ret=1 else set_var hosts_not_overidden 1 fi rm $tmpfile fi # Check for wrong server in bonding.conf # if ! has_var bonding_conf_server_correct ; then hostname=$(get_var hostname) tmpfile=$(mktemp) salt --hide-timeout --out=txt '*' cmd.run "grep -e '^server' /etc/bonding/bonding.conf | grep -v $hostname ||:" >$tmpfile 2>/dev/null if grep -q server $tmpfile ; then echo "At least one node has the wrong server hostname in /etc/bonding/bonding.conf. Please fix before continuing:" echo cat $tmpfile ret=1 else set_var bonding_conf_server_correct 1 fi rm $tmpfile fi # Check for management server ping, but only if there are no hosts entries # if has_var hosts_not_overidden ; then if ! has_var nodes_reach_management ; then hostname=$(get_var hostname) tmpfile=$(mktemp) salt --hide-timeout --out=txt '*' cmd.run "ping -c 1 $hostname ||:" >$tmpfile 2>/dev/null if grep -q "100% packet loss" $tmpfile ; then echo "At least one node cannot ping the management server by name. Please fix before continuing:" echo grep "100% packet loss" $tmpfile | cut -d ':' -f 1 ret=1 else set_var nodes_reach_management 1 fi rm $tmpfile fi fi # Check for software up-to-date # if ! has_var bondingadmin_up_to_date ; then apt-get update --allow-releaseinfo-change >/dev/null if apt list --upgradable | grep -q bondingadmin ; then echo "Bondingadmin is not up-to-date. Please upgrade before continuing" ret=1 else set_var bondingadmin_up_to_date 1 fi fi # Get API token # if ! has_var multapplied_api_token ; then set_var multapplied_api_token $(ba get multapplied_api_token) fi # Get database locale # if ! has_var db_locale ; then db_locale=$(su postgres -c "psql -Atc \"select datcollate from pg_catalog.pg_database where datname = 'bondingadmin';\"") set_var db_locale $db_locale fi if [ $ret = 0 ] ; then release=$(echo "import version; print(version.__version__)" | PYTHONPATH="/usr/lib/bondingadmin/" python3 -c "import version; print(version.__version__)") echo echo "Pre-migration checks completed. To continue, run the following on the target" echo "server in a tmux session:" echo echo " curl 'https://git.multapplied.net/Partner/bondingadmin-install/archive/master.tar.gz' | tar -xz" echo " cd bondingadmin-install" echo " ./migrate target-pre $(get_var hostname)" echo fi # Get config entries for the target install # for name in full_name short_name email country province city ; do set_var $name "get_bondingadmin_option $name" done return $ret } function check_target() { ret=0 if ! has_var bondingadmin_installed ; then if dpkg -l bondingadmin | grep -q bondingadmin ; then set_var bondingadmin_installed 1 else echo "Bondingadmin is not installed" ret=1 fi fi if ! has_var bondingadmin_up_to_date ; then apt-get update >/dev/null if apt list --upgradable | grep -q bondingadmin ; then echo "Bondingadmin is not up-to-date. Please upgrade before continuing" ret=1 else set_var bondingadmin_up_to_date 1 fi fi # Check that locale matches the source # locale=$(su postgres -c "psql -Atc \"select datcollate from pg_catalog.pg_database where datname = 'bondingadmin';\"") source_locale=$(get_var source_locale) if [ "$locale" != "$source_locale" ] ; then echo "The database locale does not match the source server. You will need to" echo "recreate with the matching locale." echo answer=$(ask_value "Do you want to recreate now? (y/n)" validate_yn) if [ "$answer" = "y" ] ; then echo "$source_locale" > /etc/locale.gen locale-gen update-locale LANG="$source_locale" LANGUAGE="$source_locale" systemctl stop postgresql rm -rf /var/lib/postgresql/* /etc/postgresql/* apt-get install -y postgresql --reinstall pg_createcluster --locale "$source_locale" --start 11 main systemctl start postgresql /usr/share/bondingadmin/initdb.sh echo "Database recreated" else echo "Please recreate the database and run this command again" return 1 fi fi return $ret } function sync_influx() { if [ "$(get_var sync_influx)" = "y" ] ; then echo "Syncing influxdb data" rsync -a $(get_var hostname):/var/lib/influxdb/ /var/lib/influxdb/ else echo "Setting up influxdb" /usr/lib/bondingadmin/influxconfig fi } usage_source="Run pre-migration actions on the source host" function action_source() { check_source } usage_target_pre=" Run pre-migration actions on the target host" function action_target_pre() { hostname=$1 if [ -z $hostname ] ; then echo "ERROR: No host name given" return 1 fi # Check to see if we can access the target host # if [ -z $SSH_AUTH_SOCK ] ; then echo "No SSH agent detected. Please log back in with agent forwarding" return 1 fi if [ -z $TMUX ] ; then echo "Please run this in a tmux session" return 1 fi # Sync the datastore from the source host # rsync -a $hostname:/var/lib/bondingadmin-migrate/ /var/lib/bondingadmin-migrate/ # Make sure the hostname matches # hostnamectl set-hostname $(get_var hostname) # Set the source IP # set_var source_ip $(python3 -c "import socket; print(socket.gethostbyname('$hostname'))") if ! has_var target_installed ; then cat <" echo return 1 fi if [ -z $TMUX ] ; then echo "Please run this in a tmux session" return 1 fi source_ip=$(get_var source_ip) # Stop the services on the new host systemctl stop bondingadmin # Remove the CA directory on the new host if it exists rm -rf /var/lib/bondingadmin/ca # Stop the services on the old host echo -e "Stopping bondingadmin bondingadmin-salt-minion and salt-master on old host" ssh "root@${source_ip}" "systemctl stop bondingadmin bondingadmin-salt-minion bondingadmin-salt-master mgmtvpn" # Run backup on old host echo -e "Running backup-bondingadmin on old host" ssh "root@${source_ip}" "backup-bondingadmin" # Copy backups from old server to new server echo -e "Copying backups from old host to new host" currentdate=$(date +%Y-%m-%d) mkdir -p "/var/lib/bondingadmin/backups/" scp "root@${source_ip}:/var/lib/bondingadmin/backups/*.${currentdate}.*" "/var/lib/bondingadmin/backups/" # Restore backup on new host echo -e "Running restore-bondingadmin on new (current) host" restore-bondingadmin # Enable all services except aggfail echo -ne "Enable all services except aggfail" systemctl enable bondingadmin bondingadmin-uwsgi homestead systemctl start bondingadmin systemctl stop aggfail set_var target_complete echo echo "Main migration completed. Next steps are:" echo echo " 1. Update the A/AAAA DNS records as appropriate" echo " 2. Check HTTPS TLS certificate" echo " 2. Wait for nodes to connect to the management server" echo " 3. Ensure nodes are accessible via salt:" echo echo " salt '*' test.ping" echo echo "Once complete continue with the post-migration using the following command:" echo echo " ./migrate target-post" echo } usage_target_post="Run post-migration actions on the target host" function action_target_post() { if ! has_var target_complete ; then echo "Main migration not complete. Please run the following command:" echo echo " ./migrate target" echo return 1 fi if [ -z $TMUX ] ; then echo "Please run this in a tmux session" return 1 fi source_ip=$(get_var source_ip) echo -e "Enabling aggfail service" systemctl enable --now aggfail echo -e "Syncing bonding repositories" ba sync_bonding_release echo -e "Regenerating bonding ISOs" ba regenerate_isos echo -e "Running backup-bondingadmin" backup-bondingadmin echo -e "Running sendmgmtpublickey" ba sendmgmtpublickey echo -e "Running remotebackup" ba remotebackup echo -e "Restarting firewall service" systemctl restart firewall # Migrate influxDB echo -ne "Migrating influxDB " if [ "$(get_var sync_influx)" = "y" ] ; then # stop influxdb on old host echo -e "Stopping influxdb on old host" ssh "root@${source_ip}" "systemctl stop influxdb" # stop influxdb on new host echo -e "Stopping influxdb on current host" systemctl stop influxdb # rsync influxdb from old to new host echo -e "rsyncing influxdb from old to new host" rsync -a "root@${source_ip}:/var/lib/influxdb/" "/var/lib/influxdb" # start influxdb on new host echo -e "Starting influxdb on current host" systemctl start influxdb fi } function get_actions() { for fn in $(declare -F | grep -e '\baction_' | cut -d ' ' -f 3) ; do action=${fn#action_} echo ${action//_/-} done } usage_actions=" Get a list of defined actions" function action_actions() { echo $(get_actions) } usage_help=" Get help for action" function action_help() { if [ -z "$1" ] ; then echo "No action specified" return 1 fi action=${1//-/_} action_function="action_$action" if ! function_defined $action_function ; then echo "$1 is not an action" return 1 fi usage_var="usage_$action" echo "$1 ${!usage_var}" help_function="help_$action" if function_defined $help_function ; then echo $help_function fi } usage_set="