#!/usr/bin/env python #=========================================================================== # SVN properties (DO NOT CHANGE) # # $Id$ # $HeadURL: $ # $LastChangedRevision$ # $Author$ # $LastChangedDate$ # #============================================================================ import socket import struct import logging import logging.handlers from string import split, strip, find, upper, lower from sys import argv, exit from os import popen, getenv, path from glob import glob from xml.parsers import expat from copy import deepcopy from time import asctime program = 'errormon2' author = 'Walter Brisken + ?' version = '0.6' verdate = '20130508' def usage(prog): print '%s ver. %s %s %s\n' % (program, version, author, verdate) print 'Usage: %s [options] []\n' % prog print 'options can include:' print ' --help' print ' -h print help information and quit\n' print ' is the max alert error level to print\n' ############################################################################## # Parameters ############################################################################## logpath = '.' logname = 'log' logcount = 100 # Number of files to keep # 0=fatal, 1=severe, 2=error, 3=warning, 4=info, 5=verbose, 6=debug stdout_level = 4 logfile_level = 6 # see help(logging.Formatter) logfile_format = logging.Formatter("%(asctime)s %(name)-10s%(levelname)-8s%(message)s") stdout_format = logging.Formatter("%(asctime)s %(name)-10s%(levelname)-8s%(message)s") ############################################################################### # Set up logger and levels corresponding to DiFXAlert levels ############################################################################### # Unfortunately python logger error levels count in the opposite direction # DiFXAlert level 0 corresponds to python 80 etc: alertLevels = [80, 70, 60, 50, 40, 30, 20, 10] logging.addLevelName(80, 'FATAL') logging.addLevelName(70, 'SEVERE') logging.addLevelName(60, 'ERROR') logging.addLevelName(50, 'WARNING') logging.addLevelName(40, 'INFO') logging.addLevelName(30, 'VERBOSE') logging.addLevelName(20, 'DEBUG') logging.addLevelName(10, 'IGNORE') # The following is just to overwrite the python DEBUG level. logging.addLevelName(10, 'VERBOSE_DEBUG') # Set up a single logger difxLogger = logging.getLogger('DiFXAlert') log = difxLogger.log # This is currently unused: # Set up different loggers for different kinds of difxmessage. All will write # to the same loggers defined below but with a different %(name) string. # By convention these are set up hierarchically with dot notation. # We could have one per machine e.g. host1.DifxLoadMessage etc. logs = {'DifxLoadMessage': logging.getLogger('DifxLoadMessage'), 'DifxErrorMessage': logging.getLogger('DifxErrorMessage'), 'Mark5StatusMessage': logging.getLogger('Mark5StatusMessage'), 'DifxStatusMessage': logging.getLogger('DifxStatusMessage'), 'DifxInfoMessage': logging.getLogger('DifxInfoMessage'), 'DifxWeightMessage': logging.getLogger('DifxWeightMessage'), 'DifxCommand': logging.getLogger('DifxCommand')} ############################################################################### # Set up logging to stdout ############################################################################### stdout_handler = logging.StreamHandler() stdout_handler.setLevel(alertLevels[stdout_level]) stdout_handler.setFormatter(stdout_format) difxLogger.addHandler(stdout_handler) ############################################################################### # Set up logging to file ############################################################################### try: file_handler = logging.handlers.RotatingFileHandler(path.join(logpath, logname), backupCount = logcount) file_handler.setFormatter(logfile_format) file_handler.setLevel(alertLevels[logfile_level]) file_handler.doRollover() difxLogger.addHandler(file_handler) except: # Adding exc_info=True writes the python exception to the log log(alertLevels[3], "Error starting log file. Only log to stdout", exc_info=True) ############################################################################### # Set up other loggers. See ############################################################################### ############################################################################### # Virtually the same as errormon ############################################################################### class Parser: def __init__(self): self._parser = expat.ParserCreate() self._parser.StartElementHandler = self.start self._parser.EndElementHandler = self.end self._parser.CharacterDataHandler = self.data self.message = '' self.severity = -1 self.tmp = '' self.ok = False self.unit = '' self.mpiid = -1 self.id = '' self.tag = '' def feed(self, data): self._parser.Parse(data, 0) def close(self): try: self._parser.Parse("", 1) # end of data del self._parser # get rid of circular references except expat.ExpatError: print 'Error parsing :', self.message, " len=", len(self.message) exit(0) def start(self, tag, attrs): self.tag = tag if tag == 'difxAlert': self.ok = True self.tmp = '' def end(self, tag): if tag == 'alertMessage' and self.ok: self.message = self.tmp if tag == 'severity': self.severity = int(self.tmp) if tag == 'from': self.unit = lower(self.tmp) if tag == 'identifier': self.id = self.tmp if tag == 'mpiProcessId': self.mpiid = int(self.tmp) def data(self, data): if self.tag == 'alertMessage': self.tmp = self.tmp + data else: self.tmp = data def getinfo(self): if self.ok: return self.severity, 'MPI[%2d] %-9s %-12s %s' % (self.mpiid, self.unit, self.id, self.message) else: return self.severity, '' def run(): port = getenv('DIFX_MESSAGE_PORT') if port == None: print 'DIFX_MESSAGE_PORT needs to be defined' exit(0) else: port = int(port) group = getenv('DIFX_MESSAGE_GROUP') if group == None: print 'DIFX_MESSAGE_GROUP needs to be defined' exit(0) s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 768000) s.bind(('', port)) mreq = struct.pack("4sl", socket.inet_aton(group), socket.INADDR_ANY) s.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) #s.settimeout(dt) try: while 1: try: message = s.recv(1500) if len(message) > 0 and message[0] == '<': p = Parser() p.feed(message) level, info = p.getinfo() p.close() if len(info) > 0: log(alertLevels[level], info) except socket.timeout: pass except expat.ExpatError: print asctime(), '*** Unparsable message received ***' print message exit(0) except KeyboardInterrupt: pass if len(argv) > 1: if argv[1] in ['-h', '--help']: usage(argv[0]) exit(0) run()