#
# Test case for failover of gfmd by rebooting server
#
load "gfarm_v2::check_gfmd_master"
load "gfarm_v2::check_gfmd_slave"
load "gfarm_v2::wait_for_boot"
load "gfarm_v2::failback"
load "gfarm_v2::check_journal_seqnum_max"
load "gfarm_v2::check_gfmd_connected"

#
# Test parameters
#
# RETRY_COUNT:	  retry number to check master/slave and boot
# RETRY_INTERVAL: retry interval to check master/slave and host to boot
# TIMEOUT:	  timeout sec for starting/stopping gfmd
#
setup_failover_by_reboot()
{
	# default parameter
	RETRY_COUNT=${RETRY_COUNT-"10"}
	RETRY_INTERVAL=${RETRY_INTERVAL-"5"}
	TIMEOUT=${TIMEOUT-"10"}

	# prepare: check whether required hosts are defined
	$GFSERVICE systest::gfarm_v2::check_hostids_defined - \
		"gfmd1 gfmd2 gfmd3 client1"
	[ $? -ne 0 ] && return $RESULT_UNSUPPORTED

	return $RESULT_PASS
}

test_failover_by_reboot()
{
	RESULT=$RESULT_PASS

	# step: reboot gfmd1
	$GFSERVICE systest::gfarm_v2::reboot_host gfmd1
	[ $? -ne 0 ] && log_warn "$TESTNAME: failed to reboot gfmd1"

	# wait for gfmd1 down here, otherwise failover fails due to:
	# <err> [1003839] cannot transform to the master gfmd because another master gfmd is already running
	sleep 5

	# step: promote gfmd on gfmd2
	$GFSERVICE promote-gfmd gfmd2
	[ $? -ne 0 ] && log_warn "$TESTNAME: failed to promote gfmd on gfmd2"

	# step: check whether gfmd on gfmd2 is master
	check_gfmd_master gfmd2 client1 $RETRY_COUNT $RETRY_INTERVAL
	if [ $? -ne 0 ]; then
		log_warn "$TESTNAME: gfmd2 in not master"
		set_result RESULT $RESULT_FAIL
	fi

	# step: wait gfmd1 to be rebooted
	wait_for_boot gfmd1 $RETRY_COUNT $RETRY_INTERVAL

	# step: set gfmd on gfmd1 to be slave
	$GFSERVICE -t $TIMEOUT start-gfarm-slave gfmd1
	[ $? -ne 0 ] && log_warn "$TESTNAME: failed to start gfmd on gfmd1" \
		"as slave"

	# step: check whether gfmd on gfmd1 is slave
	check_gfmd_slave gfmd1 client1 $RETRY_COUNT $RETRY_INTERVAL
	if [ $? -ne 0 ]; then
		log_warn "$TESTNAME: gfmd1 in not slave"
		set_result RESULT $RESULT_FAIL
	fi

	# step: failback
	failback gfmd1 gfmd2 client1 $TIMEOUT $RETRY_COUNT
	[ $? -ne 0 ] && log_warn "$TESTNAME: failed to make gfmd on gfmd1" \
		"as master"

	# step: check whether gfmd on gfmd1 is master
	check_gfmd_master gfmd1 client1 $RETRY_COUNT $RETRY_INTERVAL
	if [ $? -ne 0 ]; then
		log_warn "$TESTNAME: gfmd1 in not master"
		set_result RESULT $RESULT_FAIL
	fi

	# step: check whether slave gfmd is connected to master
	check_gfmd_connected gfmd2 client1 $RETRY_COUNT $RETRY_INTERVAL
	if [ $? -ne 0 ]; then
		log_warn "$TESTNAME: gfmd2 is not connected to master"
		set_result RESULT $RESULT_FAIL
	fi

	check_gfmd_connected gfmd3 client1 $RETRY_COUNT $RETRY_INTERVAL
		if [ $? -ne 0 ]; then
		log_warn "$TESTNAME: gfmd3 is not connected to master"
		set_result RESULT $RESULT_XFAIL
	fi

	# step: check whether journal seqnum max is same
	check_journal_seqnum_max gfmd1 gfmd2
	if [ $? -ne 0 ]; then
		log_warn "$TESTNAME: journal seqnum max is not same"
		set_result RESULT $RESULT_FAIL
	fi

	return $RESULT
}

teardown_failover_by_reboot()
{
	return $RESULT_PASS
}
