# -*- fill-column: 80; -*-

import os
import time
import logging
import traceback
try:
    from setproctitle import setproctitle
except ImportError:
    def setproctitle(*args):
        pass
try:
    from systemd.daemon import notify as sd_notify
except ImportError:
    def sd_notify(*args):
        pass

############################################################

from ._version import __version__
from . import const
from . import db
from .worker import Worker

############################################################

class GuardDaemonError(Exception):
    pass

class Daemon(object):
    """Guardian daemon.

    """
    def __init__(self, system, logger,
                 initial_op='EXEC',
                 initial_mode=None,
                 initial_state=None,
                 initial_request=None,
                 single_shot=False,
                 archive=None,
                 ):
        # check arguments
        if initial_op not in db.DAEMON_OP:
            raise GuardDaemonError("Initial OP not in %s" % str(DAEMON_OP))
        if initial_mode and initial_mode not in db.DAEMON_MODE:
            raise GuardDaemonError("Initial MODE ({}) not in {}".format(initial_mode, str(DAEMON_MODE)))

        # FIXME: check that system is initialized already
        self.system = system
        self.log = logger
        self.redirect_timeout = const.REDIRECT_TIMEOUT
        self.single_shot = single_shot

        # set the default loglevel to be the initialized loglevel
        self.default_loglevel = logging.getLevelName(self.log.getEffectiveLevel())

        self.log.info("Guardian v%s", __version__)

        if self.single_shot:
            self.log.info("single-shot mode")

        # worker subprocess
        self.worker = None

        # store redirect request time
        # FIXME: this is the only internal state variable that is not
        # an exposed PV.  Should it be?
        self.redirect_request_time = None

        # EPICS channel access server thread
        if self.single_shot:
            self.cas = None
        else:
            from .cas import CAServer
            self.cas = CAServer(self.system.name)
            self.log.info("EPICS control prefix: %s", self.cas.prefix)

        # archive git repo
        if archive:
            self.archive = archive
            self.log.info("system archive: %s" % (self.archive.root))
        else:
            self.archive = None

        # initialize internal db
        self.db = db.Database(self.cas)

        # initialize state index and request enum
        self.db.update_system(self.system, init=True)

        # set initial values
        self['TIME_INIT'] = int(time.time())
        self['OP'] = initial_op
        if initial_mode is not None:
            self['MODE'] = initial_mode
        elif self.system.manager:
            self['MANAGER'] = self.system.manager
            self['MODE'] = 'MANAGED'
        else:
            self['MODE'] = 'AUTO'
        self['LOGLEVEL'] = self.default_loglevel
        self['STATE'] = initial_state or 'INIT'
        self['TARGET'] = self['STATE']
        self['REQUEST'] = initial_request or self.system.request or 'NONE'
        self['NOMINAL'] = self.system.nominal or 'NONE'
        self['SPM_MONITOR'] = self.system.ca_monitor
        self['SPM_MONITOR_NOTIFY'] = self.system.ca_monitor_notify
        self['STATUS'] = 'INIT'

        self.archive_commit('DAEMON INIT')
        self.archive_print()

        self.log.info("system name: %s", self.system.name)
        self.log.info("system CA prefix: %s", self.system.ca_prefix)
        self.log.info("module path: %s", self.system.path)
        if self.system.usercode:
            for code in self.system.usercode:
                self.log.info("user code: %s", code)
        self.log.debug("system states: %s", self.system.states)
        self.log.debug("request states: %s", self.system.request_states)
        self.log.info("initial state: %s" % self['STATE'])
        self.log.info("initial request: %s" % self['REQUEST'])
        if not self.single_shot:
            self.log.info("nominal state: %s" % self['NOMINAL'])
        if self.system.manager:
            self.log.info("initial manager: %s" % self['MANAGER'])
        self.log.info("CA setpoint monitor: %s" % self['SPM_MONITOR'])
        self.log.info("CA setpoint monitor notify: %s" % self['SPM_MONITOR_NOTIFY'])

        self.log.info("daemon initialized")

    def single_shot_exit(self, code=0, message=None):
        if self.single_shot:
            if code != 0 and message:
                self.log.error(message)
            raise SystemExit(code)

    ########################################
    # archive

    def archive_print(self):
        if not self.archive:
            return
        hexid = self.archive.get_hexsha()
        intid = self.archive.get_intsha()
        self.log.info("system archive: id: %s (%d)" % (hexid, intid))

    def archive_commit(self, message):
        if not self.archive:
            return
        message += '''

guardian version: {}
'''.format(__version__)
        ret = False
        with self.archive as archive:
            if archive.commit(self.system, message):
                self.log.info("system archive: code changes detected and committed")
                ret = True
        self['ARCHIVE_ID'] = self.archive.get_intsha()
        return ret

    ########################################
    # "magic" methods for accessing/setting the process variables

    def __getitem__(self, channel):
        return self.db[channel]

    def __setitem__(self, channel, value):
        if self.db[channel] != value \
           and db.guarddb[channel].get('log_changes', True):
            self.log.debug('%s => %s' % (channel, value))
        self.db[channel] = value

    ########################################

    def message(self, message):
        self.log.info(message)
        self['GRDMSG'] = message

    ########################################

    def worker_init(self):
        self.log.debug("initializing worker...")
        snapshot_file = None
        if self.archive:
            snapshot_file = os.path.join(self.archive.root, '%s.spm' % self.system.name)
        self.worker = Worker(self.system, self.log, snapshot_file)
        self.worker.start()
        self.log.info("worker ready")

    def worker_terminate(self):
        if self.worker is not None and self.worker.is_alive():
            self.log.debug("terminating worker...")
            self.worker.terminate()
            self.worker.join()
            self.log.info("worker terminated")
        self.worker = None

    ########################################
    ########################################

    def unstall(self):
        if self['STALLED']:
            self.log.info("STALL cleared")
            self['STALLED'] = False

    def raise_redirect(self):
        self['STATUS'] = 'REDIRECT'
        self.message("REDIRECT requested, timeout in %.3f seconds" % (self.redirect_timeout))
        self.redirect_request_time = time.time()

    def update_target(self, last_request):
        # this function updates the TARGET state based on the current
        # STATE and REQUEST.  It also potentially updates STATUS and
        # ERROR as needed.

        # in MANUAL mode, simply set target and redirect immediately
        if self['MODE'] == 'MANUAL' and self['REQUEST'] != 'NONE':
            self['TARGET'] = self['REQUEST']
            # FIXME: option to not redirect here, to let state finish?
            self.raise_redirect()
            return

        # request confirmations
        if self['REQUEST'] != last_request:
            # special INIT request handling: for INIT requests,
            # redirect to INIT immediately but reset the request back
            # to last request.
            if self['REQUEST'] == 'INIT':
                self.message("INIT redirect")
                self['REQUEST'] = last_request
                self['TARGET'] = 'INIT'
                self.raise_redirect()
                return

            # if changing from NONE requests to a request that
            # corresponds to the current state don't raise the normal
            # state re-request redirect.
            if self['REQUEST'] == self['STATE'] and last_request == 'NONE':
                self.log.info("request set from NONE to current state ({}); doing nothing.".format(self['STATE']))
                return

        # if the request is NONE, do nothing
        if self['REQUEST'] == 'NONE':
            # FIXME: if a NONE request comes in during a
            # non-requestable state, the state will continually be run
            # in the DONE status, even though that might not be
            # desirable
            self['TARGET'] = self['STATE']
            return

        # calculate shortest path from current state to current request.
        self.log.info("calculating path: %s->%s" % (self['STATE'], self['REQUEST']))
        newpath = self.system.shortest_path(self['STATE'], self['REQUEST'])

        # if path is None, there is no path so clean up and return
        if newpath is None:
            self.message("no path from %s->%s" % (self['STATE'], self['REQUEST']))
            self['TARGET'] = self['STATE']
            self.single_shot_exit(5)
            return

        self.log.debug("calculated path: %s" % newpath)

        # the second element of the path list is the target, or the
        # first element if the current state is the target.  if it's
        # different than the current target update the target
        if newpath[:2][-1] != self['TARGET']:
            self['TARGET'] = newpath[:2][-1]
            self.log.info("new target: %s" % self['TARGET'])

        # The following deals with state REDIRECTS.  a redirect occurs
        # when a new request results in a goto target state.  in this
        # case we don't wait for the current state to complete and
        # instead move directly to the goto state.  we also use a
        # redirect when the state equals the request and the user
        # requests the current state to be rerun.  we *don't* use a
        # redirect when there is a jump to a state with a goto target,
        # or the current state is INIT, as we want to make sure those
        # states are fully executed first before transitioning.
        #
        # a timeout in seconds is used to indicate that the worker
        # process should be forcibly killed if it has not returned in
        # the specified amount of time. this is harsh, since
        # terminating the worker destroys the worker process and the
        # existing EPICS (ezca) IO object, requiring the worker to be
        # restarted and all EPICS connections to be reestablished.

        # no redirects if we're stalled
        if self['STALLED']:
            return

        # first describe the in-coming and out-going edges to make the
        # logic clearer (True indicates a normal edge transition)
        #
        # inedge: False, True, 'jump'
        if self['STATUS'] == 'EDGE':
            inedge = True
        elif self['STATUS'] == 'JUMP':
            inedge = 'jump'
        else:
            inedge = False
        # outedge: False, True, 'goto'
        if self['TARGET'] == self['STATE']:
            outedge = False
        elif self.system.is_goto(self['TARGET']) and self['STATE'] != 'INIT':
            outedge = 'goto'
        else:
            outedge = True
        # now handle the redirect condition itself
        if outedge == 'goto':
            if not self.system.can_redirect(self['STATE']):
                self.message("GOTO REDIRECT IGNORED: redirect=False for state %s" % self['STATE'])
            else:
                self.message("GOTO REDIRECT")
                self.raise_redirect()
        elif self.single_shot:
            pass
        # same-state redirect only if state has edge to itself
        elif inedge == outedge == False and self.system.graph.has_edge(self['STATE'], self['STATE']):
            self.log.info("same state request redirect")
            self.raise_redirect()

    def reload_system(self):
        self.message("RELOAD requested.  reloading system data...")
        self.system.load()
        request_list_change = self.db.update_system(self.system)
        self.log.info("module path: %s", self.system.path)
        if self.system.usercode:
            for code in self.system.usercode:
                self.log.info("user code: %s", code)
        self.log.debug("system states: %s", self.system.states)
        self.log.debug("request states: %s", self.system.request_states)
        newnom = self.system.nominal or 'NONE'
        if newnom != self['NOMINAL']:
            self['NOMINAL'] = newnom
            self.log.info("new nominal state: %s" % self['NOMINAL'])
        if self.system.ca_monitor != self['SPM_MONITOR']:
            self['SPM_MONITOR'] = self.system.ca_monitor
            self.log.info("CA setpoint monitor: %s" % self['SPM_MONITOR'])
        if self.system.ca_monitor_notify != self['SPM_MONITOR_NOTIFY']:
            self['SPM_MONITOR_NOTIFY'] = self.system.ca_monitor_notify
            self.log.info("CA setpoint monitor notify: %s" % self['SPM_MONITOR_NOTIFY'])
        if self.archive_commit('DAEMON RELOAD'):
            self.archive_print()
        if request_list_change:
            self.message("RELOAD complete; REQUEST LIST CHANGED: *RELOAD MEDM CONTROL SCREEN*")
        else:
            self.message("RELOAD complete")

    ########################################
    ########################################

    def stop(self):
        # FIXME: what actions do we need to do here?
        sd_notify("STOPPING=1")
        self.log.info("stopping daemon...")
        self.worker_terminate()
        self.log.info("daemon stopped.")

    ########################################

    def run(self):
        self.log.info("============= daemon start =============")

        setproctitle('guardian {} {}'.format(self.system.name, self.system.path))

        # internal flags for noting transitions
        op = None
        mode = None
        manager = ''
        request = self['REQUEST']

        # holds time at which to reset log level
        loglevel_reset_time = None

        # handle times in int usec, to prevent cumulative round off error
        time_step = int(1000000/const.CPS)
        # start on the next integer second
        next_step = int(time.time() + 1) * 1000000 - time_step

        # start the worker subprocess
        self.worker_init()

        self.log.info("========== executing run loop ==========")
        loopstart = time.time()
        sd_notify("READY=1")

        ########################################
        ########################################
        while True:

            self['MAINUSEC'] = int((time.time() - loopstart)*1000000)

            # FIXME: this would be the best place to call this, as this should
            # only need to be called once to update all the values pushed during
            # the last part of the loop, as opposed to in the process loop
            # below, but see comment in cas.updatePVs.
            #if self.cas:
            #    self.cas.updatePVs()

            # sleep/process till next cycle
            next_step += time_step
            while int(time.time()*1000000) < next_step:
                if self.cas:
                    # NOTE: See comment in cas.upatePVs().  The best place I've
                    # found to put it is here, right before the server process
                    # happens, although I can't say exactly why.  If called too
                    # frequently (i.e. after every variable set or with the
                    # sleep time of this loop set too low) the daemon process
                    # will be in R more than S, jacking the system load up
                    # considerably.  If we put it here but set the process time
                    # too high, there's more skew in the total loop processing
                    # time.
                    self.cas.updatePVs()

                    # The main part of the loop below usually completely in less
                    # than 5ms, so there should be ~58ms in this process loop.
                    # A 5ms sleep here seems to be a reasonable compromise with
                    # the updatePVs constraints described above.  (Could adjust
                    # time as it gets closer.)
                    self.cas.process(0.005)
                else:
                    time.sleep(0.001)

            self.log.log(5, '====== start cycle ======')
            looplast = loopstart
            loopstart = time.time()
            self['LOOPUSEC'] = int((loopstart - looplast)*1000000)
            sd_notify("WATCHDOG=1")

            ########################################
            # bookeeping to be done every cycle

            # update heartbeat
            self['TIME_UP'] = int(time.time() - self['TIME_INIT'])

            # update the database to pull in any CA changes
            self.db.update()

            # note OP and MODE changes
            if self['OP'] != op:
                self.message("OP: %s" % self['OP'])
                op = self['OP']
            if self['MODE'] != mode:
                self.message("MODE: %s" % self['MODE'])
                mode = self['MODE']

            # check for worker failures
            if self.worker is not None and self.worker.exitcode is not None:
                raise GuardDaemonError("worker exited unexpectedly, exit code: %d" % self.worker.exitcode)

            # twiddle the log level
            if self['LOGLEVEL'] != logging.getLevelName(self.log.getEffectiveLevel()):
                self.log.setLevel(self['LOGLEVEL'])
                self.worker['LOGLEVEL'] = self['LOGLEVEL']
                self.log.info('LOGLEVEL => %s' % logging.getLevelName(self.log.getEffectiveLevel()))
                # reset log level in 1 minute
                loglevel_reset_time = time.time() + 60
            # reset back to default level after timeout
            if loglevel_reset_time and time.time() >= loglevel_reset_time:
                self['LOGLEVEL'] = self.default_loglevel
                self.log.setLevel(self['LOGLEVEL'])
                self.worker['LOGLEVEL'] = self['LOGLEVEL']
                loglevel_reset_time = None

            # retrieve status from worker
            if self.worker:
                # status
                self['WORKER'] = self.worker['STATUS']
                # notifications
                nmsgs = self.worker['NUSERMSGS']
                self['NOTIFICATION'] = nmsgs > 0
                # HACK: we use the "raw" value from the shmem
                # interface because of pcaspy bugginess when a char
                # array is set to an empty string.  The "raw" value is
                # actually an array of null bytes the same size as the
                # record, which seems to make things happier.
                for i in range(const.USERMSG_COUNT):
                    self['USERMSG'+str(i)] = self.worker._shmem['USERMSG'+str(i)].raw
                # floor to USERMSG_COUNT so 'ind' doesn't end up
                # greater than the number of USERMSG slots
                if nmsgs > const.USERMSG_COUNT:
                    nmsgs = const.USERMSG_COUNT
                if nmsgs > 0:
                    ind = str(int((time.time() / const.USERMSG_CYCLE_TIME) % nmsgs))
                    self['USERMSG'] = self.worker._shmem['USERMSG'+ind].raw
                else:
                    self['USERMSG'] = self.worker._shmem['USERMSG0'].raw
                # PV/setpoint monitoring
                self['PV_TOTAL'] = self.worker['PV_TOTAL']
                self['SPM_TOTAL'] = self.worker['SPM_TOTAL']
                self['SPM_CHANGED'] = self.worker['SPM_CHANGED']
                # HACK: see shmem hack note above
                for i in range(const.SPM_DIFF_COUNT):
                    self['SPM_DIFF'+str(i)] = self.worker._shmem['SPM_DIFF'+str(i)].raw
                    self['SPM_DIFF'+str(i)+'_S'] = self.worker._shmem['SPM_DIFF'+str(i)+'_S'].raw
                    self['SPM_DIFF'+str(i)+'_C'] = self.worker._shmem['SPM_DIFF'+str(i)+'_C'].raw
                    self['SPM_DIFF'+str(i)+'_D'] = self.worker._shmem['SPM_DIFF'+str(i)+'_D'].raw
                # PV snapshots
                if self['SPM_SNAP']:
                    self.worker.snapshot_request.set()
                    self['SPM_SNAP'] = False
                # manager status
                self['SUBNODES_TOTAL'] = self.worker['SUBNODES_TOTAL']
                self['SUBNODES_NOT_OK'] = self.worker['SUBNODES_NOT_OK']

            # log manager changes
            if self['MANAGER'] != manager:
                if self['MANAGER'] == '':
                    self.log.warning("MANAGER: removed")
                    self.unstall()
                    # FIXME: this should prompt a target update
                elif manager == '':
                    self.log.warning("MANAGER: assigned: %s", self['MANAGER'])
                else:
                    self.log.warning("MANAGER: reassigned, new manager: %s" % self['MANAGER'])
                manager = self['MANAGER']

            ##########
            # SUMMARY BITS

            self['ACTIVE'] = \
                self['OP'] == 'EXEC' \
                and \
                self['MODE'] in ['AUTO', 'MANAGED'] \
                and \
                self['LOAD_STATUS'] == 'DONE' \
                and \
                self['WORKER'] != 'INIT' \
                and \
                not self['ERROR'] \
                and \
                self['CONNECT'] == 'OK'

            self['READY'] = \
                self['ACTIVE'] \
                and \
                self['SUBNODES_NOT_OK'] == 0

            self['INTENT'] = \
                self['ACTIVE'] \
                and \
                self['REQUEST'] == self['NOMINAL'] \
                and \
                not self['STALLED']

            self['OK'] = \
                self['READY'] \
                and \
                self['INTENT'] \
                and \
                self['STATE'] == self['NOMINAL'] \
                and \
                self['STATUS'] == 'DONE'

            ########################################
            # handle code reload requests

            if self['LOAD']:
                self['LOAD_STATUS'] = 'REQUEST'
                self.message("LOAD REQUEST")
                self['LOAD'] = False
                try:
                    self.reload_system()
                    self.worker.load_request.set()
                    self['LOAD_STATUS'] = 'SET'
                    if self['ERROR']:
                        self['ERROR'] = False
                except:
                    self.log.error(traceback.format_exc())
                    self.message("LOAD ERROR: see log for more info (LOAD to reset)")
                    self['LOAD_STATUS'] = 'ERROR'
                    self['ERROR'] = True

            ########################################
            # transitons and target updates
            # but not if there's been a load error

            if not self['ERROR']:

                # update state on transition
                # status is updated only *after* path target is updated
                if self['STATUS'] in ['EDGE', 'JUMP']:
                    self.log.info("%s: %s->%s" % (self['STATUS'], self['STATE'], self['TARGET']))
                    self['STATE'] = self['TARGET']

                # calculate path and update target (this is dependent on the
                # details of the in and out edges)
                if self.db.request_event:
                    self.message("REQUEST: %s" % self['REQUEST'])
                    self.unstall()
                    self.update_target(request)
                    self.db.request_event = False
                    request = self['REQUEST']
                elif self['STATUS'] in ['INIT', 'EDGE', 'JUMP'] and self['MODE'] != 'MANUAL':
                    self.update_target(request)

                # status update only after target calculation
                if self['STATUS'] in ['INIT', 'EDGE', 'JUMP']:
                    self['STATUS'] = 'ENTER'

            ########################################
            ########################################
            ########################################

            self.log.log(5, '--------- %-4s  ---------' % self['OP'])

            # updatePVs here, roughly half way through the loop, so that the few
            # channels that change during both halfs of the loop will have their
            # all updates exposed.
            if self.cas:
                self.cas.updatePVs()

            ########################################
            # on STOP terminate the worker and restart it, reset the
            # state, and go to pause

            if self['OP'] == 'STOP':
                self['STATUS'] = 'INIT'
                self.worker_terminate()
                self.worker_init()
                if self['EXECTIME'] != 0:
                    self['EXECTIME_LAST'] = self['EXECTIME']
                self['OP'] = 'PAUSE'
                self['LOAD'] = True

            ########################################
            ########################################
            # each EXEC cycle attempt to acquire worker run lock.
            # acquiring it means the worker is currently idle,
            # allowing us to inspect the worker status and execute the
            # next function.

            if self.worker.acquire():
                self.log.log(5, "worker lock acquired")

                # update the latest worker status *after* acquiring
                # the worker lock
                self['WORKER'] = self.worker['STATUS']

                self.log.log(5, "exec status: %s", self['STATUS'])
                self.log.log(5, "worker status: %s" % self['WORKER'])

                # update "last" worker cycle time info
                # do this *before* the ERROR continue below
                if self.worker['EXECSTOP'] > self.worker['EXECSTART']:
                    self['EXECTIME'] = self.worker['EXECSTOP'] - self.worker['EXECSTART']
                    self['EXECTIME_LAST'] = self['EXECTIME']
                self['EXECCYCLES_LAST'] = self['EXECCYCLES']

                # abort any execution if we're in error
                if self['ERROR']:
                    self.worker.release()
                    self.log.log(5, "worker lock released")
                    continue

                # only reset the exec times if we're not in error
                self['EXECTIME'] = 0
                self['EXECCYCLES'] = 0

                # handle load transitions
                if self['LOAD_STATUS'] == 'SET':
                    self['LOAD_STATUS'] = 'INPROGRESS'
                elif self['LOAD_STATUS'] == 'INPROGRESS':
                    self['LOAD_STATUS'] = 'DONE'

                # clear connection errors if worker no longer
                # returning CERROR
                if self['CONNECT'] == 'ERROR' and self['WORKER'] != 'CERROR':
                    self.message("connections reestablished")
                    self['CONNECT'] = 'OK'

                ##########
                # EXECUTION EVALUTATION

                # if we're paused do nothing
                if self['OP'] == 'PAUSE':
                    pass

                # if we're entering a state, execute MAIN
                elif self['STATUS'] == 'ENTER':
                    self.message("executing state: %s (%s)" % (self['STATE'], self.system.index(self['STATE'])))
                    self['STATUS'] = 'MAIN'

                # if this is a redirect, transition to the redirect
                # target, or redo the current state
                elif self['STATUS'] == 'REDIRECT':
                    self.message("REDIRECT caught")
                    self.redirect_request_time = None
                    self.log.info("[%s.redirect]" % self['STATE'])
                    if self['TARGET'] == self['STATE']:
                        self['STATUS'] = 'ENTER'
                    elif self['MODE'] == 'MANUAL':
                        self['STATUS'] = 'JUMP'
                    else:
                        self['STATUS'] = 'EDGE'

                # otherwise evaluate the worker return status and make a decision
                elif self['STATUS'] in ['MAIN', 'RUN', 'DONE']:

                    # worker INIT indicates a worker restart, in which case we
                    # don't need to update the status
                    if self['WORKER'] == 'INIT':
                        pass

                    elif self['WORKER'] == 'CONTINUE':
                        self['STATUS'] = 'RUN'

                    elif self['WORKER'] == 'DONE':
                        if self['STATE'] == self['TARGET'] or self['STALLED']:
                            self['STATUS'] = 'DONE'
                            self.single_shot_exit(0, "[%s.exit]" % self['STATE'])
                        else:
                            self['STATUS'] = 'EDGE'

                    elif self['WORKER'] == 'JUMP':
                        self.message("JUMP target: %s" % self.worker['JUMP'])
                        if self['MODE'] == 'MANUAL':
                            self['STATUS'] = 'DONE'
                        else:
                            self.log.info("[%s.exit]" % self['STATE'])
                            self['STATUS'] = 'JUMP'
                            self['TARGET'] = self.worker['JUMP']
                        self.single_shot_exit(10)
                        # if MANAGED, jump transitions cause the node to STALL,
                        # meaning that EDGEs will not be followed until the
                        # stall is cleared by a REQUEST.
                        if self['MODE'] == 'MANAGED':
                            self.message('STALLED')
                            self['STALLED'] = True

                    elif self['WORKER'] == 'CERROR':
                        self.single_shot_exit(2, "EZCA CONNECTION ERROR, exiting.")
                        # do this check so we only log on new error
                        if self['CONNECT'] != 'ERROR':
                            self['CONNECT'] = 'ERROR'
                            self.message("EZCA CONNECTION ERROR. attempting to reestablish...")
                            self.log.error("CERROR: State method raised an EzcaConnectionError exception.")
                            self.log.error("CERROR: Current state method will be rerun until the connection error clears.")
                            self.log.error("CERROR: If CERROR does not clear, try setting OP:STOP to kill worker, followed by OP:EXEC to resume.")

                    elif self['WORKER'] == 'ERROR':
                        # if we're in the middle of a load caused by worker
                        # error, the main error would have been cleared during
                        # the initial reload, which would then drop us through
                        # to the execution phase here, even though the worker
                        # status error would still be present
                        if self['LOAD_STATUS'] == 'INPROGRESS':
                            pass

                        else:
                            self.message("ERROR in state %s: see log for more info (LOAD to reset)" % self['STATE'])
                            self['ERROR'] = True
                            self.single_shot_exit(1, "ERROR, exiting.")

                    # other worker return status ('COMMAND') should not occur
                    # and constitutes an internal error
                    else:
                        raise GuardDaemonError("Internal error: invalid WORKER in loop: %s" % self['WORKER'])

                # any other status ('INIT', 'EDGE', 'JUMP') should not occur and
                # constitutes an internal error
                else:
                    raise GuardDaemonError("Internal error: invalid STATUS in loop: %s" % self['STATUS'])

                ##########
                # execute method

                if self['ERROR']:
                    pass

                elif self['OP'] == 'PAUSE':
                    self.worker.command(self['STATE'], 'NOOP')

                elif self['STATUS'] == 'MAIN':
                    self.worker.command(self['STATE'], 'MAIN')

                elif self['STATUS'] in ['RUN', 'DONE']:
                    self.worker.command(self['STATE'], 'RUN')

                ##########
                # must release run lock for worker to acquire it,
                # or to reaquire it on the next cycle

                self.worker.release()
                self.log.log(5, "worker lock released")

            ########################################
            ########################################
            # if run lock can't be acquired, the worker is active and
            # still holding it, so do nothing unless we have a
            # redirect

            else:
                self.log.log(5, "worker locked")

                self['EXECTIME'] = time.time() - self.worker['EXECSTART']
                self['EXECCYCLES'] += 1

                # if we have a redirect, wait for the timeout then
                # forcibly terminate the worker and reset it.
                if self['STATUS'] == 'REDIRECT':
                    self.message("REDIRECT wait for worker completion...")
                    if time.time() >= self.redirect_request_time + self.redirect_timeout:
                        self.message("REDIRECT timeout reached. worker terminate and reset...")
                        self.worker_terminate()
                        self.worker_init()
                        self.redirect_request_time = None
                        self['STATUS'] = 'EDGE'
