#!/usr/bin/env ruby
# Run for 'production' env by default
ENV["RAILS_ENV"] ||= 'production'
# Skip dashboard client
SKIP_DASHBOARD_CLIENT = true
$daemon_online = true

## exit if other daemon is already running
$pid_file = File.expand_path('../../tmp/pids/onapp_daemon.pid', __FILE__)
process = %x[pgrep -f '^ruby(.)*(/onapp_daemon_ctl run)'|grep -v #{Process.pid}]
pid = process.scan(/\d+/).first unless process.empty?

if File.exists?($pid_file) && defined?(pid) && !pid.nil?
  puts "Daemon is already running. [#{$pid_file}, #{pid}]"
  exit 1
end

require File.expand_path('../../config/environment.rb', __FILE__)
require 'eventmachine'
require 'lib/processor'

# Set pool size equal as connection pool
# if we will set pool size more than pool, we will have Mysql connection timeout errors
EventMachine.threadpool_size = ActiveRecord::Base.connection_config[:pool] - 5

# Use separate file for Rails logs
Rails.logger = ActiveSupport::TaggedLogging.new(Logger.new("#{Rails.root}/log/#{Rails.env}_rails_transaction_logger.log", 10, (500*1024*1024)))

$daemon_log = "#{Rails.root}/log/onapp_daemon.log"

def logger(reload = false)
  return @logger if defined?(@logger) && !reload

  @logger = Logger.new($daemon_log, 10, (500*1024*1024))
  @logger.level = Logger::INFO if ENV["RAILS_ENV"].eql?('production')
  @logger.formatter = proc do |severity, datetime, progname, msg|
    "[#{severity}][#{Process.pid}] #{datetime} #{msg}\n"
  end
  EM.watch_file($daemon_log, LoggerMonitor)
  @logger
end

module LoggerMonitor
  def file_deleted
    logger(true)
  end
end

File.unlink(Processor.status_socket_file) if File.exists?(Processor.status_socket_file)

module PidMonitor
  def file_deleted
    graceful_stop
  end
end

def graceful_stop
  logger.info("[PID #{Process.pid}] COUNTERS : #{$counters.inspect}")
  logger.info("[PID #{Process.pid}] TIMERS : #{$timers.inject({}) {|h,t| h.merge(t[0] => t[1].interval)}}")
  logger.info('STOPPING DAEMON ...')
  $daemon_online = false
  EventMachine.stop_server($server) #Stop statistics sock service
  $timers.values.each { |timer| timer.cancel } #stop periodical processes
  logger.info("Waiting for connection(s) to finish ...")
  unless wait_for_connections_and_stop # check runed processes
    EventMachine.add_periodic_timer(5) do
      wait_for_connections_and_stop
    end
  end
  logger.info('DAEMON STOPPED!')
end

def wait_for_connections_and_stop
  running_transactions = Transaction.running.where(:pid => Process.pid).pluck('transactions.id')
  logger.info("Waiting for transactions to be processed: [#{running_transactions}]") if running_transactions.any?
  return false if Transaction.running.where(:pid => Process.pid).count() > 0
  logger.info("[PID #{Process.pid}]:  All connection(s) closed. ")
  EventMachine.stop
end

def delete_pid_file(pid_file)
  logger.info('Trying to remove PID file and stop watching.')
  if File.exist?(pid_file) && File.open(pid_file).read.to_i == Process.pid
    $pid_monitor.stop_watching if $pid_monitor
    File.unlink(pid_file)
  else
    logger.info('PID file has been already removed.')
  end
end

def tagged_logger(logger, tag, message)
  logger.tagged(tag) {logger.info(message)}
end

$counters = {
    'cdn_sync_runners' => 0,
    'backup_runners' => 0,
    'transaction_runners' => 0,
    'hypervisor_monitor' => 0,
    'zombie_disk_space_updater' => 0,
    'schedule_runners' => 0,
    'cluster_monitors' => 0,
    'snmp_stats_level1' => 0,
    'snmp_stats_level2' => 0,
    'snmp_stats_level3' => 0,
    'vmware_stats_level1' => 0,
    'vmware_stats_level2' => 0,
    'solidfire_stats_level1' => 0,
    'cp_space_updater' => 0,
    'backup_server_monitor' => 0,
    'auto_scaling_runner' => 0,
    'failover_processor' => 0,
    'supplier_runners' => 0,
    'trader_runners' => 0
}

$timers = {}

EM.run {
  File.open($pid_file, 'w') { |file| file.write Process.pid }

  # Add Unix socket monitor
  $server = EM::start_unix_domain_server(Processor.status_socket_file, ServerProcessorMonitor)

  logger.info("#{Rails.env} => DAEMON START")
  logger.info("Bootstrapping up to #{EM.threadpool_size} Threads")

  Signal.trap("INT") { logger.info("SCHEDULER INT signal."); delete_pid_file($pid_file); EM.stop }
  Signal.trap("TERM") { logger.info("SCHEDULER TERM signal."); delete_pid_file($pid_file); graceful_stop }
  Signal.trap("EXIT") { logger.info("SCHEDULER EXIT!"); delete_pid_file($pid_file); EM.stop }
  #Signal.trap("HUP") {  }

  # Stop event machine if pid file has been removed
  $pid_monitor = EM.watch_file($pid_file, PidMonitor)

  EM.error_handler { |e|
    logger.error("Error raised during event loop: #{e.message}")
    logger.error("Error raised during event loop: #{e.backtrace}")
  }

  # many processes
  transaction_runner = TransactionRunnerProcessor.new
  supplier_runner = SupplierRunnerProcessor.new
  trader_runner = TraderRunnerProcessor.new
  backup_runner = BackupRunnerProcessor.new
  backup_server_monitor = BackupServerMonitor.new
  # 1 process runners
  schedule_runner = ScheduleRunnerProcessor.new
  monitis_monitor = MonitisMonitorProcessor.new
  cdn_sync_runner = CdnSyncRunnerProcessor.new
  zombie_disk_space_updater = ZombieDiskSpaceUpdaterProcessor.new
  snmp_stats_runner1 = SNMPStatsRunnerProcessor.new(:level => 1)
  snmp_stats_runner2 = SNMPStatsRunnerProcessor.new(:level => 2)
  snmp_stats_runner3 = SNMPStatsRunnerProcessor.new(:level => 3)
  vmware_stats_runner1 = VMWareStatsRunnerProcessor.new(:level => 1)
  vmware_stats_runner2 = VMWareStatsRunnerProcessor.new(:level => 2)
  solidfire_stats_runner = SolidfireStatsRunnerProcessor.new(:level => 1)
  auto_scaling_runner = AutoScalingRunnerProcessor.new
  failover_processor = FailoverProcessor.new

  $transaction_supervisor = ProcessSupervisor.new(EM.threadpool, :transaction_runner)

  EM.add_periodic_timer(60) do #check every 60 sec if log files removed and recreate
    [transaction_runner, backup_runner, backup_server_monitor, schedule_runner, monitis_monitor,
     cdn_sync_runner, zombie_disk_space_updater, snmp_stats_runner1, snmp_stats_runner2, snmp_stats_runner3, vmware_stats_runner1,
     vmware_stats_runner2, solidfire_stats_runner, auto_scaling_runner].collect { |d| d.init_logger }
  end

  ##CDN sync runner
  $timers['cdn_sync_runners'] = EM.add_periodic_timer(OnApp.configuration.cdn_sync_delay.seconds) do
    if $counters['cdn_sync_runners'] < cdn_sync_runner.max
      EM.defer(
          proc {
            $counters['cdn_sync_runners'] +=1
            begin
              cdn_sync_runner.logger.info("Starting CDN Sync Runner at #{Time.now}")
              ProcessRunner.log 'cdn_sync_runner' do
                cdn_sync_runner.run
              end
            rescue => e
              cdn_sync_runner.logger.error(e.message)
            ensure
              $counters['cdn_sync_runners'] -= 1
            end
          })
    end
  end

  # Backup runners
  $timers['backup_runners'] = EM.add_periodic_timer(OnApp.configuration.backup_taker_delay) do
    EM.defer(
        proc {
          $counters['backup_runners'] +=1
          begin
            backup_runner.logger.info("Start Backup Runner #{$counters['backup_runners']}")
            if backup_runner.can_run?
              backup_runner.run
            else
              backup_runner.logger.info('No Scheduled Backups!')
            end
          rescue => e
            backup_runner.logger.error(e.message)
          ensure
            backup_runner.logger.info('Stop Backup Runner')
            $counters['backup_runners'] -= 1
          end
        })
  end

  # Backup Server monitors
  $timers['backup_server_monitor'] = EM.add_periodic_timer(1.minute) do
    if $counters['backup_server_monitor'] <= 0
      EM.defer(
          proc {
            BackupServer.find_each(:batch_size => 50) do |bs|
              $counters['backup_server_monitor'] += 1
              begin
                backup_server_monitor.logger.info("Start Backup Server Monitor: IP #{bs.ip_address}")
                backup_server_monitor.run(bs)
              rescue => e
                backup_server_monitor.logger.error(e.message)
              ensure
                backup_runner.logger.info("Stop Backup Server Monitor IP #{bs.ip_address}")
                $counters['backup_server_monitor'] -= 1
              end
            end
          })
    end
  end

  # Transaction runners
  $timers['transaction_runners'] = EM.add_periodic_timer(OnApp.configuration.transaction_runner_delay) do
    if OnApp.configuration.simultaneous_transactions > $counters['transaction_runners']
      EM.defer do
        transaction = $transaction_supervisor.available_task
        if transaction.present?
          $counters['transaction_runners'] += 1
          begin
            transaction_runner = $transaction_supervisor.register(transaction.id)
            transaction_runner.run(transaction) if transaction_runner
          rescue => e
            $transaction_supervisor.try(:logger).try(:error,"TransactionRunner ##{transaction_runner.try(:id)}: #{e.message}")
          ensure
            runner_id = Thread.current[:runner_id]
            $transaction_supervisor.try(:unregister, runner_id) unless runner_id.nil?
            $counters['transaction_runners'] -= 1
          end
        end
      end
    end
  end

  # Supplier runners
  $timers['supplier_runners'] = EM.add_periodic_timer(OnApp.configuration.transaction_runner_delay) do
    EM.defer(
        proc {
          $counters['supplier_runners'] +=1
          begin
            supplier_runner.logger.info("Start check Supplier Runner #{$counters['transaction_runners']}")
            if supplier_runner.can_run?
              supplier_runner.run
            else
              supplier_runner.logger.info('No Scheduled Supplier actions!')
            end
          rescue => e
            supplier_runner.logger.error(e.message)
          ensure
            supplier_runner.logger.info('Stop Supplier Runner')
            $counters['supplier_runners'] -= 1
          end
        })
  end

  # Trader runners
  $timers['trader_runners'] = EM.add_periodic_timer(OnApp.configuration.transaction_runner_delay) do
    EM.defer(
        proc {
          $counters['trader_runners'] +=1
          begin
            trader_runner.logger.info("Start check Trader Runner #{$counters['transaction_runners']}")
            if trader_runner.can_run?
              trader_runner.run
            else
              trader_runner.logger.info('No Scheduled Trader actions!')
            end
          rescue => e
            trader_runner.logger.error(e.message)
          ensure
            trader_runner.logger.info('Stop Trader Runner')
            $counters['trader_runners'] -= 1
          end
        })
  end

  #CP disk usage monitoring
  # every 15 minutes we will check if space on any partiotion less 95%
  $timers['cp_space_updater'] = EM.add_periodic_timer(15.minutes) do
    if $counters['cp_space_updater'] < 1
      EM.defer(
          proc {
            $counters['cp_space_updater'] += 1
            begin
              logger.info("Start CP free space check #{$counters['cp_space_updater']}")
              hostname = `hostname`
              message = "CP #{hostname}\nDisk free space less than 95%\n"
              usage = `df -h`
              unless usage.scan(/[0-9]{1,}\%.*/).collect { |row| row.split(' ') }.select { |k, v| message << "#{v}\t#{k}\n" if k.to_i > 95 }.empty?
                Alert.log!(Alert.new, :free_space, :warn, message)
                SystemAlert.system_resources(message).deliver if OnApp.configuration.system_notification
              end
            ensure
              $counters['cp_space_updater'] -= 1
            end
          })
    end
  end

  # Zombie disk space updater
  $timers['zombie_disk_space_updater'] = EM.add_periodic_timer(OnApp.configuration.zombie_disk_space_updater_delay) do
    if $counters['zombie_disk_space_updater'] < zombie_disk_space_updater.max
      EM.defer(
          proc {
            $counters['zombie_disk_space_updater'] += 1
            begin
              zombie_disk_space_updater.logger.info("Start Zoombie Diskspace updater #{$counters['zombie_disk_space_updater']}")
              zombie_disk_space_updater.run
            ensure
              $counters['zombie_disk_space_updater'] -= 1
            end
          })
    end
  end

  # Schedule runner
  $timers['schedule_runners'] = EM.add_periodic_timer(OnApp.configuration.schedule_runner_delay) do
    if $counters['schedule_runners'] < schedule_runner.max
      EM.defer(
          proc {
            $counters['schedule_runners'] +=1
            begin
              schedule_runner.logger.info("Start Shedule runner #{$counters['schedule_runners']}")
              schedule_runner.run
            ensure
              $counters['schedule_runners'] -= 1
            end
          })
    end
  end

  # Cluster monitor
  $timers['cluster_monitors'] = EM.add_periodic_timer(OnApp.configuration.cluster_monitor_delay) do
    if $counters['cluster_monitors'] < monitis_monitor.max
      EM.defer(
          # run task
          proc {
            $counters['cluster_monitors'] += 1
            begin
              monitis_monitor.logger.info("Start Cluster monitor #{$counters['cluster_monitors'] }")
              monitis_monitor.run
            ensure
              $counters['cluster_monitors'] -= 1
            end
          })
    end
  end

  # SNMP Stats
  #level 1
  $timers['snmp_stats_level1'] = EM.add_periodic_timer(OnApp.configuration.snmp_stats_level1_period) do
    snmp_stats_runner1.logger.info("Start SNMP Stat Level 1: (HV defers: #{$counters['snmp_stats_level1']})")
    if $counters['snmp_stats_level1'] <= 0
      ProcessRunner.log 'snmp_stats_level1' do
        snmp_stats_runner1.run {
          Hypervisor.not_vmware.find_each(:batch_size => 50) do |hv|
            EM.defer(
                proc {
                  $counters['snmp_stats_level1'] += 1
                  begin
                    tagged_logger(snmp_stats_runner1.logger, "HV: #{hv.id}", "Get Level 1 for HV: #{hv.ip_address}")
                    hv.get_level1(snmp_stats_runner1.logger)
                  rescue => e
                    tagged_logger(snmp_stats_runner1.logger, "HV: #{hv.id}", e)
                  ensure
                    $counters['snmp_stats_level1'] -= 1
                  end
                })
          end rescue nil
        }
      end
    end
  end

  #level 2
  $timers['snmp_stats_level2'] = EM.add_periodic_timer(OnApp.configuration.snmp_stats_level2_period) do
    snmp_stats_runner2.logger.info("Start SNMP Stat Level 2: (HV defers: #{$counters['snmp_stats_level2']})")
    if $counters['snmp_stats_level2'] <= 0
      #$counters['snmp_stats_level2'] = Hypervisor.not_vmware.count rescue 0
      ProcessRunner.log 'snmp_stats_level2' do
        snmp_stats_runner2.run {
          Hypervisor.not_vmware.find_each(:batch_size => 50) do |hv|
            EM.defer(
                proc {
                  $counters['snmp_stats_level2'] += 1
                  begin
                    tagged_logger(snmp_stats_runner2.logger, "HV: #{hv.id}", "Get Level 2 for HV: #{hv.ip_address}")
                    hv.get_level2(snmp_stats_runner2.logger)
                  rescue => e
                    tagged_logger(snmp_stats_runner2.logger, "HV: #{hv.id}", e)
                  ensure
                    $counters['snmp_stats_level2'] -= 1
                  end
                })
          end rescue nil
        }
      end
    end
  end

  #level 3
  $timers['snmp_stats_level3'] = EM.add_periodic_timer(OnApp.configuration.snmp_stats_level3_period) do
    snmp_stats_runner3.logger.info("Start SNMP Stat Level 3: (HV defers: #{$counters['snmp_stats_level3']})")
    if $counters['snmp_stats_level3'] <= 0
      ProcessRunner.log 'snmp_stats_level3' do
        snmp_stats_runner3.run {
          Hypervisor.not_vmware.update_all(:list_of_volume_groups => '', :list_of_logical_volumes => '')
          Hypervisor.not_vmware.not_used_for_backup.hv_ids_with_uniq_data_stores.each do |hv| #should be run to get lvdisplay and vgdisplay only
            EM.defer(
                proc {
                  $counters['snmp_stats_level3'] += 1
                  begin
                    tagged_logger(snmp_stats_runner3.logger, "HV: #{hv.id}", "Get Level 3 for HV: #{hv.ip_address}")
                    hv.get_level3(snmp_stats_runner3.logger)
                  rescue => e
                    tagged_logger(snmp_stats_runner3.logger, "HV: #{hv.id}", e)
                  ensure
                    $counters['snmp_stats_level3'] -= 1
                  end
                })
          end
        }
      end
    end
  end

  # VMWare Stats
  #level 1
  $timers['vmware_stats_level1'] = EM.add_periodic_timer(OnApp.configuration.vmware_stats_level1_period) do
    vmware_stats_runner1.logger.info("Start VMWare Stat Level 1: (HV defers: #{$counters['vmware_stats_level1']})")
    if $counters['vmware_stats_level1'] <= 0
      ProcessRunner.log 'vmware_stats_level1' do
        vmware_stats_runner1.run {
          Hypervisor.vmware.all.each do |hv|
            EM.defer(
                proc {
                  $counters['vmware_stats_level1'] += 1
                  begin
                    tagged_logger(vmware_stats_runner1.logger, "HV: #{hv.id}", "Get Level 1 for HV: #{hv.ip_address}")
                    hv.get_level1(vmware_stats_runner1.logger)
                  rescue => e
                    tagged_logger(vmware_stats_runner1.logger, "HV: #{hv.id}", e)
                  ensure
                    $counters['vmware_stats_level1'] -= 1
                  end
                })
          end rescue nil
        }
      end
    end
  end

  #level 2
  $timers['vmware_stats_level2'] = EM.add_periodic_timer(OnApp.configuration.vmware_stats_level2_period) do
    vmware_stats_runner2.logger.info("Start VMWare Stat Level 2: (HV defers: #{$counters['vmware_stats_level2']})")
    if $counters['vmware_stats_level2'] <= 0
      ProcessRunner.log 'vmware_stats_level2' do
        vmware_stats_runner2.run {
          Hypervisor.vmware.all.each do |hv|
            EM.defer(
                proc {
                  $counters['vmware_stats_level2'] += 1
                  begin
                    tagged_logger(vmware_stats_runner2.logger, "HV: #{hv.id}", "Get Level 2 for HV: #{hv.ip_address}")
                    hv.get_level2(vmware_stats_runner2.logger)
                  rescue => e
                    tagged_logger(vmware_stats_runner2.logger, "HV: #{hv.id}", e)
                  ensure
                    $counters['vmware_stats_level2'] -= 1
                  end
                })
          end rescue nil
        }
      end
    end
  end

  # SolidFire stats level 1 only
  $timers['solidfire_stats_level1'] = EM.add_periodic_timer(OnApp.configuration.solidfire_stats_usage_interval) do
    solidfire_stats_runner.logger.info("SolidFire Stats: (Defers: #{$counters['solidfire_stats_level1']})")
    if $counters['solidfire_stats_level1'] <= 0
      solidfire_stats_runner.run {
        DataStore.solidfire.all.each do |ds|
          EM.defer(
              proc {
                $counters['solidfire_stats_level1'] += 1
                begin
                  solidfire_stats_runner.logger.info("Get Stats for SF DataStore: #{ds.label}##{ds.id} [#{ds.ip}]")
                  ds.get_level1
                rescue => e
                  solidfire_stats_runner.logger.info(e)
                ensure
                  $counters['solidfire_stats_level1'] -= 1
                end
              })
        end rescue nil
      }
    end
  end

  # AutoScalingRunner
  $timers['auto_scaling_runner'] = EM.add_periodic_timer(5.minutes) do
    if $counters['auto_scaling_runner'] < auto_scaling_runner.max
      EM.defer(
          proc {
            $counters['auto_scaling_runner'] +=1
            begin
              auto_scaling_runner.logger.info("Start AutoScaling runner #{$counters['auto_scaling_runner']}")
              auto_scaling_runner.run
            ensure
              $counters['auto_scaling_runner'] -= 1
            end
          })
    end
  end

  # FailoverProcessor
  $timers['failover_processor'] = EM.add_periodic_timer(60.seconds) do
    if $counters['failover_processor'] < failover_processor.max
      EM.defer(
        proc {
          $counters['failover_processor'] += 1
          begin
            if failover_processor.can_run?
              failover_processor.logger.info("Start Failover Processor #{$counters['failover_processor']}")
              failover_processor.run
            end
          ensure
            $counters['failover_processor'] -= 1
          end
        }
      )
    end
  end

}
