#!/bin/sh
# Author: Michail Flouris (C) 2014

# restartlivevdiskbackends <Vdisk UUID>
if [ $# -ne 1 ] ; then  # Must have ONE command-line args
	echo "Usage: $0 <vdisk uuid> # of vdisk to pause and restart nbd(s) and rspamd(s)"
	exit -1
fi

vdiskid=$1
PRESCRIPT="/tmp/pre-${vdiskid}-cmds.sh"
POSTSCRIPT="/tmp/post-${vdiskid}-cmds.sh"

rm -f $PRESCRIPT $POSTSCRIPT || exit -1
touch $PRESCRIPT $POSTSCRIPT || exit -1
chmod +x $PRESCRIPT $POSTSCRIPT || exit -1

#Step 0 - Check provided vdisk uuid, if vdisk is in sync and if frontends are active locally...
#===============================================================================================
echo -n "Querying diskinfo for vdisk $vdiskid : "
diskinfo=$(/usr/bin/onappstore diskinfo uuid\=$vdiskid)

#check for error or non-existing vdisk
if [ $? != 0 ] || [[ $diskinfo != *result\=SUCCESS* ]] ; then
	echo "ERROR : vdisk $vdiskid not found, or diskinfo failure!"
	exit -1
fi

if [[ $diskinfo != *\ status\=1* ]] || [[ $diskinfo != *\ snapshot\=0* ]] ; then
	echo "ERROR : vdisk $vdiskid is not online, or is a snapshot vdisk!"
	exit -1
fi

fecnt=`ps axw | grep bdevclient | grep $vdiskid | grep -v grep | wc -l`
fecount=$(($fecnt))
if [ $? -ne 0 ]; then echo "ps axw cmd failed! Exiting!" ; fi

fulluuid=`ps axw | grep bdevclient | grep $vdiskid | grep -v grep | awk '{print $12}' | uniq`
fecntuni=`echo $fulluuid | wc -l`
fecountuni=$(($fecntuni))

if [ $fecount -lt 1 ]; then
	echo "$0 : no local frontends found for vdisk with uuid $vdiskid, is the vdisk online locally??"
	exit -1
fi
if [ $fecountuni -ne 1 ]; then
	echo "$0 : Error : multiple vdisks with uuid $vdiskid found, please choose unique one"
	exit -1
fi

if [ $fulluuid != $vdiskid ]; then
	echo "$0 : Error : Vdisk uuid $vdiskid is partial, please provide full uuid $fulluuid ??"
	exit -1
fi

#Step 1 - Check running rspamds on backend nodes and record PIDs and config files
#=================================================================================
sshcmd="/usr/bin/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=quiet"

nodeips=`ps axw | grep bdevclient | grep $vdiskid | awk '{print $6}' | sort | uniq | xargs`
nodeiparr=($nodeips) # array
echo "Nodes: $nodeips"

TMPFILE="/tmp/killrspamds-`uuidgen | cut -d'-' -f1`"
nodeipcnt="${#nodeiparr[@]}"
for idx in ${!nodeiparr[*]}; do
	n=${nodeiparr[$idx]}
	$sshcmd $n "ps w | grep $vdiskid | grep rspamd | grep -v grep" > $TMPFILE
	rspamdnpids[$idx]=`cat $TMPFILE | grep nbd | awk '{print $1}' | xargs`
	rspamdmpids[$idx]=`cat $TMPFILE | grep main | awk '{print $1}' | xargs`
	rspamdndids[$idx]=`cat $TMPFILE | grep main | awk '{print $8}' | tr -d '[]' | xargs`
	rm -f $TMPFILE
	echo "Node : $n -> MAIN PIDS: ${rspamdmpids[$idx]} - WORKER PIDS: ${rspamdnpids[$idx]}"
	rnp=( ${rspamdnpids[$idx]} )
	rmp=( ${rspamdmpids[$idx]} )
	if [ ${#rmp[@]} -ne ${#rnp[@]} ] ; then
		echo "Different number of workers vs. parent rspamds on node $n, exiting!" 
		exit -1
	fi
	echo "Node : $n -> IDs: ${rspamdndids[$idx]}"
	$sshcmd $n sync

# NOTE: CONFIG FILES NOT NEEDED, activations are done via the API
# check and make list of existing config files for the vdisks on this node
#	rspamdconfs[$idx]=""
#	for id in ${rspamdndids[$idx]}; do
#		vd=`echo $id | cut -f1 -d\:`
#		nid=`echo $id | cut -f2 -d\:`
#		cfile="/DB/NODE-$nid/vdisks/${vd}.xml"
#		
#		cfileexists=$((`$sshcmd $n ls -1 $cfile | wc -l`))
#		if [ $cfileexists -eq 1 ] ; then
#			#echo "OK, FOUND CONFIG  FILE $cfile FOR $id"
#			true
#		else
#			echo "ERROR, NOT FOUND CONFIG FILE $cfile FOR $id - ABORTING!"
#			exit -1
#		fi
#	done
done

#Step 2 - Generate deactivate cmds for online vdisk with uuid and frontends on this HV and generate prescript
#============================================================================================================

echo "sync;sync" >> $PRESCRIPT
echo /sbin/dmsetup suspend $vdiskid >> $PRESCRIPT
dmdevs=`ls -1 /dev/mapper/*_${vdiskid} | xargs`
dmdevsar=($dmdevs) # array
dmdevsx=`ls -1 /dev/mapper/*_${vdiskid}`
dmdevcount=$(( `echo $dmdevs | wc -l` ))

echo "sync;sync" >> $PRESCRIPT
#for dm in $dmdevs; do echo /sbin/dmsetup suspend `basename ${dm}` >> $PRESCRIPT; done

#Step 3 - Generate kill rspamd cmds for backend nodes in prescript
#===================================================================

for idx in ${!nodeiparr[*]}; do
	n=${nodeiparr[$idx]}
	echo "$sshcmd $n sync" >> $PRESCRIPT
	echo "echo Terminating Main Rspamd pids: ${rspamdmpids[$idx]}" >> $PRESCRIPT
	echo "$sshcmd $n kill -15 ${rspamdmpids[$idx]} 1>/dev/null 2>&1" >> $PRESCRIPT
	echo "echo Terminating Worker Rspamd pids: ${rspamdnpids[$idx]}" >> $PRESCRIPT
	echo "$sshcmd $n kill -15 ${rspamdnpids[$idx]} 1>/dev/null 2>&1" >> $PRESCRIPT
	echo "$sshcmd $n sync" >> $PRESCRIPT
	echo "rnpids=( \`$sshcmd $n ps w | grep $vdiskid | grep rspamd | grep nbd | grep -v grep | awk '{print \$1}' | xargs\` )" >> $PRESCRIPT
	echo "rmpids=( \`$sshcmd $n ps w | grep $vdiskid | grep rspamd | grep main | grep -v grep | awk '{print \$1}' | xargs\` )" >> $PRESCRIPT
	echo "if [ \${#rnpids[@]} -ne 0 ] || [ \${#rmpids[@]} -ne 0 ] ; then" >> $PRESCRIPT
	echo "echo KILLING Remaining Main Rspamd pids: \${!rmpids[*]}" >> $PRESCRIPT
	echo "$sshcmd $n kill -9 \${!rmpids[*]} 1>/dev/null 2>&1" >> $PRESCRIPT
	echo "echo KILLING Remaining Worker Rspamd pids: \${!rnpids[*]}" >> $PRESCRIPT
	echo "$sshcmd $n kill -9 \${!rnpids[*]} 1>/dev/null 2>&1" >> $PRESCRIPT
	echo "$sshcmd $n sync ; sleep 1" >> $PRESCRIPT
	echo "fi" >> $PRESCRIPT
done
echo "sleep 3" >> $PRESCRIPT

#NOTE: DEACTIVATE NOT WORKING -> Step 2 - use dbdeactivate API cmds (state 15) to terminate rspamds and nbds...
#================================================================================================================

# using state 15 for deactivateLocal!
#echo mkdir -p /tmp/curl-${vdiskid}-script >> $PRESCRIPT
#echo rm -f /tmp/curl-${vdiskid}-script/* >> $PRESCRIPT
#nbddevs=`ps axw | grep bdevclient | grep $vdiskid | grep -v grep | awk '{print $8}'`
#nbddevar=($nbddevs) # array
#for nbd in $nbddevs; do
#    nbdarray=(`/usr/bin/nbd_conn_info $nbd| awk '{gsub(/:/," ");gsub(/\//," ");gsub(/-/," ");print}'`)
#    IP=${nbdarray[4]}
#    NODE=${nbdarray[10]}
#    VDISK=${nbdarray[12]}
#	echo "echo Deactivating $VDISK on $NODE @ $IP" >> $PRESCRIPT
#    DS=`find /dev | grep ${VDISK}| grep -vE 'disk|udev|mapper' |cut -d'/' -f3`
#    echo "curl -o /tmp/curl-${vdiskid}-script/${VDISK} -X PUT http://${IP}:8080/is/Datastore/${DS}/VDisk/${VDISK} -d '{\"state\":\"15\", \"member\":\"${NODE}\", \"transaction_id\":\"0\"}' 1>/dev/null 2>&1" >> $PRESCRIPT
#done

#Step 4 - Produce activateLocalVDisk API commands to be executed prior to the nbd connections being started
#===========================================================================================================

# using state 14 for activateLocal!
echo mkdir -p /tmp/curl-${vdiskid}-script >> $POSTSCRIPT
nbddevs=`ps axw | grep bdevclient | grep $vdiskid | grep -v grep | awk '{print $8}'`
nbddevar=($nbddevs) # array
for nbd in $nbddevs; do
    nbdarray=(`/usr/bin/nbd_conn_info $nbd| awk '{gsub(/:/," ");gsub(/\//," ");gsub(/-/," ");print}'`)
    IP=${nbdarray[4]}
    NODE=${nbdarray[10]}
    VDISK=${nbdarray[12]}
    DS=`find /dev | grep ${VDISK}| grep -vE 'disk|udev|mapper' |cut -d'/' -f3`
	echo "echo Activating $VDISK on $NODE @ $IP" >> $POSTSCRIPT
    echo "curl -o /tmp/curl-${vdiskid}-script/${VDISK} -X PUT http://${IP}:8080/is/Datastore/${DS}/VDisk/${VDISK} -d '{\"state\":\"14\", \"member\":\"${NODE}\", \"transaction_id\":\"0\"}' 1>/dev/null 2>&1" >> $POSTSCRIPT
done
echo "sleep 2" >> $POSTSCRIPT

#Step 5 - Activate all nbd devices and unpause all frontends again in post script
#=================================================================================

TMPFILE="/tmp/activevdisks-`uuidgen | cut -d'-' -f1`"
ps axw | grep bdevclient | grep $vdiskid | grep -v grep | awk '{sub(/.*onapp/,"/onapp");print}' > $TMPFILE
while read line; do
    nbddev=`echo ${line} | awk '{print $4}'`
    echo "nbd_conn_info ${nbddev} | grep 'connected=1'; if [ \$? != 0 ]; then ${line}; fi" >> $POSTSCRIPT
done < ${TMPFILE}
rm -f ${TMPFILE}

#for dm in $dmdevs; do echo /sbin/dmsetup resume `basename ${dm}` >> $POSTSCRIPT; done
echo /sbin/dmsetup resume $vdiskid >> $POSTSCRIPT
echo "sync;sync" >> $PRESCRIPT


#Step 6 - End the transactions created while activating the VDisks (in post script )
#====================================================================================

for nbd in $nbddevs; do
    nbdarray=(`/usr/bin/nbd_conn_info $nbd| awk '{gsub(/:/," ");gsub(/\//," ");gsub(/-/," ");print}'`)
    IP=${nbdarray[4]}
    NODE=${nbdarray[10]}
    VDISK=${nbdarray[12]}
    DS=`find /dev | grep ${VDISK}| grep -vE 'disk|udev|mapper' |cut -d'/' -f3`
    echo "curl -o /tmp/curl-${vdiskid}-script/${VDISK} -X PUT http://${IP}:8080/is/Datastore/${DS}/VDisk/${VDISK} -d '{\"state\":\"12\", \"member\":\"${NODE}\", \"transaction_id\":\"0\"}' 1>/dev/null 2>&1" >> $POSTSCRIPT
done
echo rm -Rf /tmp/curl-${vdiskid}-script >> $POSTSCRIPT

#Step 7 - now run the pre-script and kill rspamds on back end
#============================================================================================================
echo " "
echo "================================================================"
echo "RUNNING STAGE 1: Pausing vdisk frontend and stopping backends"
source $PRESCRIPT
echo " "
echo "STAGE 1: DONE"


#Step 8 - now run the post-script
#============================================================================================================
echo "==================================================================="
echo "RUNNING STAGE 2: Restarting backends and unpausing vdisk frontend"
echo " "
source $POSTSCRIPT
echo " "
echo "STAGE 2: DONE"

