#!/bin/sh

set -e
#set -x

usage () {
        echo "$0 <CLUSTER-NAME>"
        echo "This command restores the state of a backuped PoC cluster."
        echo "See oci-poc-save to save the state of a cluster."
        exit 1
}

if ! [ $# = 1 ] ; then
        usage
fi
if [ "${1}" = "-h" ] || [ "${1}" = "--help" ] || [ "${1}" = "-help" ] ; then
	usage
fi

CLUSTER_NAME=${1}

if ! [ -d /var/lib/openstack-cluster-installer-poc/saved/${CLUSTER_NAME} ] ; then
	echo "${CLUSTER_NAME} doesn't exist"
	exit 1
fi

PID_DIR=/var/run/oci-poc


wait_for_ssh () {
	local COUNT CYCLES OTCI_CAN_SSH SSH_HOST
	SYSUSERNAME=root
	# This is 15 minutes
	COUNT=900
	CYCLES=0
	OTCI_CAN_SSH=no
	SSH_HOST=${1}

	echo -n "---> Attempting to connect to ${SSH_HOST}: "
	ssh-keygen -f ~/.ssh/known_hosts -R ${SSH_HOST} 1>/dev/null 2>/dev/null || true
	while [ "${OTCI_CAN_SSH}" != "yes" ] && [ ${COUNT} != 0 ] ; do
		if ssh -o "StrictHostKeyChecking no" -o "ConnectTimeout 2" ${SYSUSERNAME}@${SSH_HOST} 'echo -n ""' 2>/dev/null ; then
			OTCI_CAN_SSH=yes
			echo ok.
		else
			COUNT=$(( ${COUNT} - 1 ))
			CYCLES=$(( ${CYCLES} + 1 ))
			sleep 1
			echo -n "."
		fi
	done
	ssh -o "StrictHostKeyChecking no" -o "ConnectTimeout 2" ${SYSUSERNAME}@${SSH_HOST} 'echo -n ""' 1>/dev/null 2>/dev/null
}

kill_all_running_vm () {
	echo "===> Killing all VMs and IPMI SIM..."
	for i in $(find /var/run/oci-poc/ -iname '*.pid') ; do
		kill $(cat $i) || true
		rm -f $i
	done
}

restore_all_vm_from_hdd () {
	echo "===> Restoring backups of .qcow2 files..."
	cp -v /var/lib/openstack-cluster-installer-poc/saved/${CLUSTER_NAME}/*.qcow2 /var/lib/openstack-cluster-installer-poc/runtime/
	cp -v /var/lib/openstack-cluster-installer-poc/saved/${CLUSTER_NAME}/*.conf /var/lib/openstack-cluster-installer-poc/ipmi_sim/
}

start_all_vms () {
	echo "===> Starting VMs..."
	echo "---> Starting PXE server"

	start-stop-daemon \
		--start \
		--quiet \
		--background \
		--pidfile ${PID_DIR}/pxe-server-node.pid.ipmisim.pid \
		--make-pidfile \
		--startas /usr/bin/ipmi_sim \
		--      -n \
			-c /var/lib/openstack-cluster-installer-poc/ipmi_sim/pxe-server-node.conf \
			-f /etc/oci-poc/ipmisim1.emu

	echo "---> Waiting 20 seconds for the PXE server to be up..."
	sleep 20

	VM_NUM_LIST=$(for i in $(ls /var/lib/openstack-cluster-installer-poc/ipmi_sim/slave-node-*.conf) ; do echo $(basename $i) | cut -d- -f3 | cut -d. -f1; done | sort -V | tr '\n' ' ')
	for i in ${VM_NUM_LIST} ; do
		echo "---> Starting-up VM $i"
		start-stop-daemon \
			--start \
			--quiet \
			--background \
			--pidfile ${PID_DIR}/slave-node-${i}.pid.ipmisim.pid \
			--make-pidfile \
			--startas /usr/bin/ipmi_sim \
			--      -n \
				-c /var/lib/openstack-cluster-installer-poc/ipmi_sim/slave-node-${i}.conf \
				-f /etc/oci-poc/ipmisim1.emu
		sleep 1
	done
}

wait_for_ssh_of_pxe_and_controllers () {
	wait_for_ssh oci
	for i in $(ocicli -csv machine-list -s | q -H -d, "SELECT Cur_ip FROM - WHERE role='controller'") ; do
		wait_for_ssh $i
	done
}

start_galera_service () {
	FIRST_NODE=yes
	echo "===> Starting Galera cluster"
	for i in $(ocicli -csv machine-list -s | q -H -d, "SELECT Cur_ip FROM - WHERE role='controller'") ; do
		if [ "${FIRST_NODE}" = "yes" ] ; then
			echo "---> Starting galera_new_cluster on host $i"
			ssh $i "sed -i 's/safe_to_bootstrap: 0/safe_to_bootstrap: 1/' /var/lib/mysql/grastate.dat"
			ssh $i galera_new_cluster || true
			if ! ssh $i "[ -e /var/run/mysqld/mysqld.pid ]" ; then
				echo "---> Failed: trying a 2nd time on host $i"
				ssh $i galera_new_cluster || true
				if ! ssh $i "[ -e /var/run/mysqld/mysqld.pid ]" ; then
					echo "---> Failed: trying a 3nd time on host $i"
					ssh $i galera_new_cluster || true
					if ! ssh $i "[ -e /var/run/mysqld/mysqld.pid ]" ; then
						echo "Giving up..."
						exit 1
					fi
				fi
			fi
			FIRST_NODE=no
		else
			echo "---> Starting MySQL on host $i"
			ssh $i "systemctl start mysql"
		fi
		echo "---> Waiting 2 seconds"
		sleep 2
	done
}

restart_all_services () {
	echo "===> Restarting all services on controllers"
	for i in $(ocicli -csv machine-list -s | q -H -d, "SELECT hostname,Cur_ip FROM - WHERE role='controller'") ; do
		HOST_NAME=$(echo $i | cut -d, -f1)
		HOST_IP=$(echo $i | cut -d, -f2)
		for j in apache2 aodh-api aodh-evaluator aodh-notifier barbican-api glance-api magnum-api magnum-conductor neutron-api neutron-l3-agent neutron-openvswitch-agent neutron-rpc-server nova-api nova-conductor nova-novncproxy nova-placement-api nova-scheduler octavia-api octavia-health-manager octavia-worker ; do
			echo "---> Restarting $j on ${HOST_NAME}"
			ssh ${HOST_IP} "if [ -x /etc/init.d/$j ] ; then /etc/init.d/$j stop || true ; sleep 1 ; /etc/init.d/$j start || true ; fi"
		done
	done

	echo "===> Restarting all services on network nodes"
	for i in $(ocicli -csv machine-list -s | q -H -d, "SELECT hostname,Cur_ip FROM - WHERE role='network'") ; do
		HOST_NAME=$(echo $i | cut -d, -f1)
		HOST_IP=$(echo $i | cut -d, -f2)
		for j in neutron-openvswitch-agent neutron-dhcp-agent neutron-metadata-agent neutron-l3-agent ; do
			echo "---> Restarting $j on ${HOST_NAME}"
			ssh ${HOST_IP} "if [ -x /etc/init.d/$j ] ; then /etc/init.d/$j stop || true ; sleep 1 ; /etc/init.d/$j start || true ; fi"
		done
	done

	echo "===> Restarting all services on compute nodes"
	for i in $(ocicli -csv machine-list -s | q -H -d, "SELECT hostname,Cur_ip FROM - WHERE role='compute'") ; do
		HOST_NAME=$(echo $i | cut -d, -f1)
		HOST_IP=$(echo $i | cut -d, -f2)
		for j in nova-compute cinder-volume neutron-openvswitch-agent neutron-dhcp-agent neutron-metadata-agent neutron-l3-agent ; do
			echo "---> Restarting $j on ${HOST_NAME}"
			ssh ${HOST_IP} "if [ -x /etc/init.d/$j ] ; then /etc/init.d/$j stop || true ; sleep 1 ; /etc/init.d/$j start || true ; fi"
		done
	done
}

restart_all_vms () {
	echo "===> Rebooting all VMs of the cluster"
	CTLIP=$(ocicli -csv machine-list -s | q -H -d, "SELECT Cur_ip FROM - WHERE role='controller'" | head -n 1)
	for i in $(ssh ${CTLIP} ". oci-openrc ; openstack server list --all-projects --format value -c ID") ; do
		echo "---> Issuing: openstack server reboot $i"
		ssh ${CTLIP} ". oci-openrc ; openstack server reboot $i"
	done
}

kill_all_running_vm
restore_all_vm_from_hdd
start_all_vms
wait_for_ssh_of_pxe_and_controllers
start_galera_service
restart_all_services
restart_all_vms
