#!/usr/pythoncontroller/python

import sys
import testutil 
import time
import os
import testhelpers
import json 
from subprocess import *
from threading import Thread, current_thread

def parseNameValuePairs(list):
    args = {}
    for arg in list:
        if arg.find('=') != -1:
            args[arg.split('=')[0]] = arg.split('=')[1]
    return args

def parseargs(listOfArgs):
    return parseNameValuePairs(listOfArgs[1:])

def help():
    print "Usage: update_iso_images [master_mac=<MASTER_MAC>] " \
        "[master_drive=<MASTER_DRIVE>] [master_isopath=<path_to_master_iso_file>] " \
        "[slave_macs=<comma separated slave MV macs>] " \
        "[slave_drives=<comma separated list of USB drives>] [slave_isopath=<path_to_slave_iso_file>] "
    sys.exit()

def ensureRequiredParamsPresent(dict, required_keys):
    for key in required_keys:
        if not dict.has_key(key):
            help()
            sys.exit()

def getFormat(message):
    return time.strftime('%d-%m-%Y-%H:%M:%S ', time.gmtime()) + ': %d : %s : ' % (os.getpid(), current_thread().name) + message

def getStorageNodeIp(mv_hwmac):
    ip = ''
    found = False
    while not found:
        cmd = ['/space/virtx/bin/mvctl', '-i', 'eth1', '--mvmac', mv_hwmac, '--mod-list']
        print ' '.join(cmd)
        rc, out, err = testutil.runCommand(cmd)
        print rc, out, err
        for module in out.split('\n'):
            print module
            if module.find('ip_address') != -1:
                module_info = parseNameValuePairs(module.split()[13:]) 
                print module_info
                if module_info.has_key('ip_address'):
                   ip = module_info['ip_address']
                   found = True 
                   break
        time.sleep(5)
    if not len(ip):
        raise Exception("Failed to extract storage node IP address for MV %s using mod-list." % mv_id)
    return ip

args = parseargs(sys.argv)

slave_macs = []
if args.has_key('slave_macs'):
    ensureRequiredParamsPresent(args, ['slave_drives', 'slave_isopath'])

    slave_macs = []
    if args.has_key('slave_macs'):
	slave_macs = args['slave_macs'].split(',')

    if args.has_key('slave_isopath'):     
	slave_isopath = args['slave_isopath']

    slave_ips = []
    for slave_mac in slave_macs:
	slave_ips.append(testutil.MAC_TO_IPMI_USER_PASS[slave_mac][0])

    slave_drives = []
    if args.has_key('slave_drives'):
	slave_drives = args['slave_drives'].split(',')
    
    index = 0
    for slave_ip in slave_ips:
	slave_mac = slave_macs[index]
	print getFormat("Powering off the slave %s with mac %s ...") % (slave_ip, slave_mac),
	out = testhelpers.poweroffHVWithMac(slave_mac)
	if out['result'] != 'SUCCESS':
	    print getFormat("Failed to power off slave %s with mac %s, error %s" % (slave_ip, slave_mac, out)),
	index += 1
	print getFormat("DONE")

if args.has_key('master_mac'):
    print getFormat("Updating the master ...")
    ensureRequiredParamsPresent(args, ['master_mac', 'master_drive', 'master_isopath'])

    master_mac = args['master_mac']
    master_hwmac = ''.join(master_mac.split(':'))
    master_ip = testutil.MAC_TO_IPMI_USER_PASS[master_mac][0]
    master_drive = args['master_drive']
    master_isopath = args['master_isopath']

    print getFormat("Starting ISO update with master %s (on drive %s)" % (master_hwmac, master_drive))
    # 1. Turn off all servers
    print getFormat("Powering off the master %s with mac %s ...") % (master_ip, master_mac),
    out = testhelpers.poweroffHVWithMac(master_mac)
    if out['result'] != 'SUCCESS':
	print getFormat("Failed to power off master %s with mac %s, error %s" % (master_ip, master_mac, out))
    print getFormat("DONE")

    print getFormat("Waiting for a minute to allow USB drives to appear when we startup ..."), 
    time.sleep(60)
    print getFormat("DONE")

    #2. Turn on the master, wait till it responds to mvctl
    print getFormat("Powering on the master %s with mac %s ...") % (master_ip, master_mac),
    out = testhelpers.poweronHVWithMac(master_mac)
    if out['result'] != 'SUCCESS':
	print getFormat("Failed to power on master %s with mac %s, error %s" % (master_hwmac, master_mac, out))
    print getFormat("DONE")

    # ensure it responds to mvctl
    print getFormat("Wait for the master %s to respond to mvctl ..., timesync ... " % master_hwmac),
    rc = 1
    time.sleep(60)
    while rc:
	rc=os.system('/space/virtx/bin/mvctl -i eth1 --mvmac %s --timesync `date +%s`' % (master_hwmac, '%s'))
	print " ... %d ..." % rc,
	cmd = ['/space/virtx/bin/mvctl', '-i', 'eth1', '--mvmac', '0x%s' % master_hwmac, '--vm-list']
	rc, out, err = testutil.runCommand(cmd)
	print " ... vm-list ... %d" % rc,
	if rc:
	    time.sleep(10)
	    print "retry at %s ..." % time.strftime('%H:%M:%S ', time.gmtime()),
    print getFormat("DONE")

    #3. Create a bridge VM to the master, make sure master is reachable
    print getFormat("Create a bridge VM to the master")
    # Ensure there is a scripts/bridge.sh file
    print getFormat("Ensure there is a bridge.sh file ..."),
    if not os.path.exists('./scripts/bridge.sh'):
	print getFormat("ERROR - Bridge script does not exist for MV %s" % master_ip)
	sys.exit(1)
    print "there is, execute the file to create a bridge VM to the master ...",

    master_storage_ip = getStorageNodeIp(master_hwmac)
    cmd =['./scripts/bridge.sh', master_hwmac, '.'.join(master_storage_ip.split('.')[0:2])]
    rc, out, err = testutil.runCommand(cmd)
    if rc:
	print getFormat("Failed to create bridge for MV %s, %s" % (master_ip, [rc, out, err]))
	sys.exit()
    print getFormat("DONE")

    #4. Log on to the master and cleanup the openstack VDisk etc
    print getFormat("Log on to the master and cleanup the openstack VDisk etc.")
    print getFormat("Wait for the storageAPI to respond on the master ..."),
    while True:
        cmd = ['curl', '-sS', '-X', 'GET', '%s:8080/is/Datastore' % master_storage_ip]
        rc, out, err = testutil.runCommand(cmd)
        if rc:
            time.sleep(1)
            print "retry at %s ..." % time.strftime('%H:%M:%S ', time.gmtime()), 
        else:
            break
    print getFormat("Done")
    datastore_info = out.strip()
    datastore_info = json.loads(datastore_info)
    for datastore in datastore_info.iterkeys():
        if datastore_info[datastore]['name'] == 'OPENSTACK_DS':
            openstack_datastore = datastore
            openstack_vdisk = datastore_info[datastore]['vdisks']
            openstack_member = datastore_info[datastore]['members']
            print getFormat("Openstack datastore %s, openstack vdisk %s, openstack member %s" % (openstack_datastore, openstack_vdisk, openstack_member))
            print getFormat("First deactivate the openstack vdisk ..."),
            cmd = ['ssh', '-q', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'BatchMode=yes', master_storage_ip, 'onappstore', 'deactivate', 'uuid=%s' % openstack_vdisk, 'member=%s' % openstack_member]
            rc, out, err = testutil.runCommand(cmd)
            if rc:
            	print getFormat("Failed to deactivate the openstack vdisk, rc - %d, out - %s, err - %s" % (rc, out, err)),
             	sys.exit(1)
            print getFormat("DONE")

            print getFormat("Kill the syslog process ..."),
            cmd = ['ssh', '-q', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'BatchMode=yes', master_storage_ip, 'killall', 'syslogd']
            rc, out, err = testutil.runCommand(cmd)
            if rc:
            	print getFormat("Failed to kill the syslog process, rc - %d, out - %s, err - %s" % (rc, out, err)),
            print getFormat("DONE")

            print getFormat("Unmount the storage mount for the openstack member ..."),
            cmd = ['ssh', '-q', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'BatchMode=yes', master_storage_ip, 'umount', '/DB/NODE-%s' % openstack_member]
            rc, out, err = testutil.runCommand(cmd)
            if rc:
            	print getFormat("Failed to unmount the storage mount for the openstack member, rc - %d, out - %s, err - %s" % (rc, out, err)),
            print getFormat("DONE")

            print getFormat("Deactivate the volume group ..."),
            cmd = ['ssh', '-q', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'BatchMode=yes', master_storage_ip, '/sbin/vgchange', '-an']
            rc, out, err = testutil.runCommand(cmd)
            if rc:
        	print getFormat("Failed to deactivate the volume group, rc - %d, out - %s, err - %s" % (rc, out, err)),
            print getFormat("DONE")

    print getFormat("Wipe the physical volume on the master ..."),
    cmdList = 'echo y | ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o BatchMode=yes %s /sbin/pvremove -f -f %s3' % (master_storage_ip, master_drive)
    process = Popen(cmdList, shell=True,stdout=PIPE, stderr=STDOUT)
    out,err = process.communicate()
    if process.returncode:
        print getFormat("Failed to wipe the physical volume, rc - %d, out - %s, err - %s" % (process.returncode, out.strip('\n'),err)),
    print getFormat("DONE")

    try:
	# mount the isos
	print getFormat("Mounting the master iso %s ..." % master_isopath),
	cmd = ['losetup', '-f', '--show', master_isopath]
	rc, out, err = testutil.runCommand(cmd)
	if rc:
	    print getFormat("Failed to mount the master iso, rc - %d, out - %s, err - %s" % (rc, out, err)),

	out = out.strip()
	master_mount = out.strip()
	print "mounted locally at %s" % master_mount
	
        #5. Sync the master iso image to the local drive on the master
	print getFormat("Sync the master iso image to the local drive %s on the master %s ..." % (master_drive, master_hwmac))
        #raw_input("All setup master_ip: %s, master_mount: %s, master_drive: %s" % (master_storage_ip, master_mount, master_drive))
	cmdList = '(dd bs=1M iflag=direct if=%s conv=sparse | gzip --fast) | (ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o BatchMode=yes %s "zcat | /opt/bin/dd of=%s iflag=fullblock bs=1M oflag=direct status=progress")' % (master_mount, master_storage_ip, master_drive)
	print cmdList,
	process = Popen(cmdList, shell=True,stdout=PIPE, stderr=STDOUT)
	out,err = process.communicate()
	if process.returncode:
	    print getFormat("Failed to sync the master iso image to the local drive %s on the master %s, rc - %d, out - %s, err - %s" % (master_drive, master_hwmac, process.returncode, out.strip('\n'),err)),
	print getFormat(" ... DONE")

	#6. Flush the drive and shutdown the master 
	print getFormat("Flush the drive and shutdown the master ..."),
	cmd = ['ssh', '-q', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'BatchMode=yes', master_storage_ip, '/sbin/blockdev', '--flushbufs', master_drive]
	rc, out, err = testutil.runCommand(cmd)
	if rc:
	    print getFormat("Failed to flush the drive, rc - %d, out - %s, err - %s" % (rc, out, err)),

        cmdList = 'kpartx -a %s' % master_mount
        print cmdList
        process = Popen(cmdList, shell=True,stdout=PIPE, stderr=STDOUT)
        out,err = process.communicate()
        print out, err

        cmdList = 'dd if=/dev/mapper/%sp3 bs=1M iflag=direct | md5sum' % master_mount.split('/')[-1]
        print cmdList
        process = Popen(cmdList, shell=True,stdout=PIPE, stderr=STDOUT)
        out,err = process.communicate()
        print out, err

        # check the md5sum for the copied drive
        cmdList = 'ssh %s \"/opt/bin/dd if=%s3 bs=1M iflag=direct | md5sum\"' % (master_storage_ip, master_drive)
        print cmdList,
        process = Popen(cmdList, shell=True,stdout=PIPE, stderr=STDOUT)
        out,err = process.communicate()
        print out, err
        
	out = testhelpers.poweroffHVWithMac(master_mac)
	if out['result'] != 'SUCCESS':
	    print getFormat("Failed to power off master %s with mac %s, error %s" % (master_hwmac, master_mac, out)),
	print getFormat("DONE")

	print "Wait for a minute ...",
	time.sleep(60)
	print "DONE"

	print getFormat("Startup the master ..."),
	out = testhelpers.poweronHVWithMac(master_mac)
	if out['result'] != 'SUCCESS':
	    print getFormat("Failed to power on master %s with mac %s, error %s" % (master_hwmac, master_mac, out)),
	print "DONE"

	# ensure it responds to mvctl
	print getFormat("Wait for the master %s to respond to mvctl ... timesync ..." % master_hwmac),
	rc = 1
	time.sleep(60)
	while rc:
	    rc=os.system('/space/virtx/bin/mvctl -i eth1 --mvmac %s --timesync `date +%s`' % (master_hwmac, '%s'))
	    print " ... %d ..." % rc,
	    cmd = ['/space/virtx/bin/mvctl', '-i', 'eth1', '--mvmac', '0x%s' % master_hwmac, '--vm-list']
	    rc, out, err = testutil.runCommand(cmd)
	    print " ... vm-list ... %d" % rc,
	    if rc:
		time.sleep(10)
		print "retry at %s ..." % time.strftime('%H:%M:%S ', time.gmtime()),
	print "DONE"
    finally:
        # cleanup the loopback mount points
        print getFormat("Trying to cleanup master mount %s" % master_mount)
        cmd = ['kpartx', '-d', master_mount]
        rc, out, err = testutil.runCommand(cmd)
        cmd = ['losetup', '-d', master_mount]
        rc, out, err = testutil.runCommand(cmd)

if len(slave_macs):
    print getFormat("Starting ISO update with slaves %s (on drives %s)" % (','.join(slave_macs), ','.join(slave_drives)))
    try:
	#6. For every slave:
	index = 0
	slave_mounts = []
	for slave_mac in slave_macs:
            slave_hwmac = ''.join(slave_mac.split(':'))
	    slave_ip = slave_ips[index]
	    # mount the isos
	    print getFormat("Mounting the slave iso %s ..." % slave_isopath),
	    cmd = ['losetup', '-f', '--show', slave_isopath]
	    rc, out, err = testutil.runCommand(cmd)
	    if rc:
		print getFormat("Failed to mount the slave iso, rc - %d, out - %s, err - %s" % (rc, out, err)),
	    print "DONE"

	    out = out.strip()
	    slave_mounts.append(out)

	    # 7. Turn on the slave and wait till it responds to mvctl 
	    print getFormat("Power on the slave %s ..." % slave_ip),
	    slave_mac = slave_macs[index]
	    out = testhelpers.poweronHVWithMac(slave_mac)
	    if out['result'] != 'SUCCESS':
		print getFormat("Failed to power on slave %s with mac %s, error %s" % (slave_ip, slave_mac, out)),
	    print "DONE"
	  
	    # ensure it responds to mvctl
	    print getFormat("Wait for the slave %s to respond to mvctl ... timesync ... " % slave_hwmac),
	    rc = 1
	    while rc:
		rc=os.system('/space/virtx/bin/mvctl -i eth1 --mvmac %s --timesync `date +%s`' % (slave_hwmac, '%s'))
		print " ... %d ..." % rc,
		cmd = ['/space/virtx/bin/mvctl', '-i', 'eth1', '--mvmac', '0x%s' % slave_hwmac, '--vm-list']
		rc, out, err = testutil.runCommand(cmd)
		print " ... vm-list ... %d" % rc,
		if rc:
		    time.sleep(10)
		    print "retry at %s ..." % time.strftime('%H:%M:%S ', time.gmtime()),
	    print "DONE"

	    #8. Create a bridge VM to the slave, make sure slave is reachable
	    slave_storage_ip = getStorageNodeIp(slave_hwmac)
	    cmd =['./scripts/bridge.sh', slave_hwmac, '.'.join(slave_storage_ip.split('.')[0:2])]
	    rc, out, err = testutil.runCommand(cmd)
	    if rc:
		print getFormat("Failed to create bridge for MV %s, %s" % (slave_ip, [rc, out, err]))
		sys.exit()
	    print "DONE"

	    #9. Sync the slave iso image to the drive on the slave 
	    slave_mount = slave_mounts[index]
	    slave_drive = slave_drives[index]
	    print getFormat("Sync the slave iso image from local mount %s to the local drive %s on the slave %s ..." % (slave_mount, slave_drive, slave_storage_ip))
	    cmdList = '(dd bs=1M iflag=direct if=%s conv=sparse | gzip --fast) | (ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o BatchMode=yes %s "zcat | dd bs=1M of=%s")' % (slave_mount, slave_storage_ip, slave_drive)
	    process = Popen(cmdList, shell=True,stdout=PIPE, stderr=STDOUT)
	    if process.returncode:
		print getFormat("Failed to sync the slave iso image from local mount %s to the local drive %s on the slave %s, rc - %d, out - %s, err - %s" % (slave_mount, slave_drive, slave_hwmac, process.returncode, out.strip('\n'),err)),
	    out,err = process.communicate()
	    print out, err
	    print getFormat("DONE")

	    #10. Flush the drive and shutdown the slave 
	    print getFormat("Flush the drive and shutdown the slave %s ..." % slave_hwmac),
	    cmd = ['ssh', '-q', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'BatchMode=yes', slave_storage_ip, '/sbin/blockdev', '--flushbufs', slave_drive]
	    rc, out, err = testutil.runCommand(cmd)
	    if rc:
		print getFormat("Failed to flush the drive and shutdown the slave, rc - %d, out - %s, err - %s" % (rc, out, err)),
	    print out, err

	    slave_mac = slave_macs[index]
	    out = testhelpers.poweroffHVWithMac(slave_mac)
	    if out['result'] != 'SUCCESS':
		print getFormat("Failed to power off slave %s with mac %s, error %s" % (slave_ip, slave_mac, out))
		sys.exit(1)
	    print "DONE"
	    index += 1
       
	#2. Turn on the master, wait till it responds to mvctl
	if args.has_key('master_mac'):
	    print getFormat("Powering on the master %s with mac %s ...") % (master_ip, master_mac),
	    out = testhelpers.poweronHVWithMac(master_mac)
	    if out['result'] != 'SUCCESS':
		print getFormat("Failed to power on master %s with mac %s, error %s" % (master_hwmac, master_mac, out))
	    print getFormat("DONE")

	    # ensure it responds to mvctl
	    print getFormat("Wait for the master %s to respond to mvctl ..., timesync ... " % master_hwmac),
	    rc = 1 
	    time.sleep(60)
	    while rc: 
		rc=os.system('/space/virtx/bin/mvctl -i eth1 --mvmac %s --timesync `date +%s`' % (master_hwmac, '%s'))
		print " ... %d ..." % rc, 
		cmd = ['/space/virtx/bin/mvctl', '-i', 'eth1', '--mvmac', '0x%s' % master_hwmac, '--vm-list']
		rc, out, err = testutil.runCommand(cmd)
		print " ... vm-list ... %d" % rc, 
		if rc: 
		    time.sleep(10)
		    print "retry at %s ..." % time.strftime('%H:%M:%S ', time.gmtime()),
	    print getFormat("DONE")
     
	print getFormat("Start up the slaves "),
	index = 0
	for slave_ip in slave_ips:
	    slave_mac = slave_macs[index]
	    slave_hwmac = ''.join(slave_mac.split(':'))
	    print "... starting %s" % slave_ip,
	    out = testhelpers.poweronHVWithMac(slave_mac)
	    if out['result'] != 'SUCCESS':
		print getFormat("Failed to power on slave %s with mac %s, error %s" % (slave_ip, slave_mac, out)),
	    index += 1
	    print "DONE",

	    # ensure it responds to mvctl
	    print getFormat("Wait for the slave %s to respond to mvctl ..., timesync ... " % slave_hwmac),
	    rc = 1 
	    time.sleep(60)
	    while rc: 
		rc=os.system('/space/virtx/bin/mvctl -i eth1 --mvmac %s --timesync `date +%s`' % (slave_hwmac, '%s'))
		print " ... %d ..." % rc, 
		cmd = ['/space/virtx/bin/mvctl', '-i', 'eth1', '--mvmac', '0x%s' % slave_hwmac, '--vm-list']
		rc, out, err = testutil.runCommand(cmd)
		print " ... vm-list ... %d" % rc, 
		if rc: 
		    time.sleep(10)
		    print "retry at %s ..." % time.strftime('%H:%M:%S ', time.gmtime()),
	    print getFormat("DONE")

	print "DONE"

	print getFormat("Now enable logging for the MVs ..."),
	if args.has_key('master_mac'):
	    slave_macs.append(master_mac) 
	for id in slave_macs:
            id = ''.join(id.split(':'))
	    cmd = ['/space/virtx/bin/mvctl', '-i', 'eth1', '--mvmac', '0x%s' % id, '--cons-adj-loglv', '-u', '4', '-l', '4'] 
	    rc, out, err = testutil.runCommand(cmd)
	    if rc:
		print getFormat("Failed to enable logging on MV %s" % id),
	    else:
		print "%s ..." % id

	print getFormat("Add a license to the openstack")
	cmd = ['scp', '-q', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'BatchMode=yes', './scripts/license.txt', '192.168.1.254:/root/.'] 
	rc, out, err = testutil.runCommand(cmd)
	if rc:
	    print getFormat("Failed to copy the license file on to the openstack, %s, %s, %s"  % (rc, out, err))

	cmd = ['ssh', '-q', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'BatchMode=yes', '192.168.1.254', '/root/osd', 'add_license', '--license_file=/root/license.txt']
	rc, out, err = testutil.runCommand(cmd)
	if rc:
	    print getFormat("Failed to add the license to the openstack, %s, %s, %s"  % (rc, out, err))
	print "DONE"
    
    finally:
        # cleanup the loopback mount points
        for slave_mount in slave_mounts:
            print getFormat("Trying to cleanup slave mount %s" % slave_mount)
            cmd = ['losetup', '-d', slave_mount]
            rc, out, err = testutil.runCommand(cmd)
        print getFormat("Cleaned up all loopback mount points")
