Source code for mentat.system

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# This file is part of Mentat system (https://mentat.cesnet.cz/).
#
# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------


"""
Mentat system management and inspection library.
"""


__author__ = "Jan Mach <jan.mach@cesnet.cz>"
__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"


import os
import re
import datetime
import collections
import subprocess

import pyzenkit.jsonconf


#
# Real-time module status codes.
#
STATUS_RT_RUNNING_OK            = 0   # Module is running or service is OK
STATUS_RT_RUNNING_PF_MISSING    = 1   # Module is running, but pid file is missing
STATUS_RT_RUNNING_PID_MISSMATCH = 2   # Module is running, but pid file contains different PID
STATUS_RT_RUNNING_INCONSISTENT  = 10  # Module is running, but pid files are inconsistent
STATUS_RT_RUNNING_FEWER         = 11  # Module is running in fewer that required instances
STATUS_RT_RUNNING_MORE          = 12  # Module is running in more that required instances
STATUS_RT_DEAD_PF_EXISTS        = 3   # Module is dead and pid file exists
STATUS_RT_DEAD_LOCK_EXISTS      = 4   # Module is dead and lock file exists
STATUS_RT_NOT_RUNNING           = 5   # Module is not running
STATUS_RT_UNKNOWN               = 6   # Module or service status is unknown

# Overall module status messages
OVERALL_MODULE_STATUS_MESSAGES = {
    STATUS_RT_RUNNING_OK:            "All modules are running OK",
    STATUS_RT_RUNNING_PF_MISSING:    "All modules are running, but all pid files are missing",
    STATUS_RT_RUNNING_PID_MISSMATCH: "All modules are running, but all pid files contain different PID",
    STATUS_RT_RUNNING_INCONSISTENT:  "All modules are running, but all pid files are inconsistent",
    STATUS_RT_RUNNING_FEWER:         "All modules are running in fewer that required instances",
    STATUS_RT_RUNNING_MORE:          "All modules are running in more that required instances",
    STATUS_RT_DEAD_PF_EXISTS:        "All modules are dead and all pid files exist",
    STATUS_RT_DEAD_LOCK_EXISTS:      "All modules are dead and all lock files exist",
    STATUS_RT_NOT_RUNNING:           "All modules are not running",
    STATUS_RT_UNKNOWN:               "Some modules are in different state, overall system state is unknown",
}

#
# Cronjob module status codes.
#
STATUS_CJ_ENABLED  = 0   # Cronjob is enabled
STATUS_CJ_DISABLED = 5   # Cronjob is not running
STATUS_CJ_UNKNOWN  = 6   # Cronjob status is unknown

# Overall cronjob status messages
OVERALL_CRONJOB_STATUS_MESSAGES = {
    STATUS_CJ_ENABLED:  "All cronjobs are enabled",
    STATUS_CJ_DISABLED: "All cronjobs are disabled",
    STATUS_CJ_UNKNOWN:  "Some cronjobs are in different state, overall system state is unknown",
}

#
# Cronjob module status codes.
#
STATUS_ONLINE  = 0   # The whole system is online
STATUS_OFFLINE = 5   # The whole system is offline
STATUS_UNKNOWN = 6   # System overall status is unknown

# Overall cronjob status messages
OVERALL_STATUS_MESSAGES = {
    STATUS_ONLINE:  "All modules are running OK and all cronjobs are enabled",
    STATUS_OFFLINE: "All modules are NOT running and all cronjobs are disabled",
    STATUS_UNKNOWN: "Some modules and cronjobs are in different state, overall system state is unknown",
}


REGEXP_MENTAT_PS     = re.compile(r'\s*(\d+)\s+([^\s]+)\s+([^\s]*(?:python3?))\s+[^\s]+(mentat-[^\s]+)(?:\s+(.*))?')
"""Regular expression for selecting Mentat related processes."""
REGEXP_MENTAT_PIDF   = re.compile(r'(.+?)(?:\.(\d+))?\.pid$')
"""Regular expression for selecting Mentat related PID files."""
REGEXP_MENTAT_CRONF  = re.compile(r'(.+)\.cron$')
"""Regular expression for selecting Mentat related log files."""
REGEXP_MENTAT_LOGF   = re.compile(r'(.+)\.log$')
"""Regular expression for selecting Mentat related log files."""
REGEXP_MENTAT_RLOGF  = re.compile(r'(.+)(?:\.(\d+))?\.runlog$')
"""Regular expression for selecting Mentat related runlog files."""
REGEXP_MENTAT_CACHEF = re.compile(r'(.+)\.json$')
"""Regular expression for selecting Mentat related cache files."""


[docs]class MentatModule(): """ Class representing Mentat real-time module configuration for control utility. """ def __init__(self, params): if 'exec' not in params: # raise pyzenkit.baseapp.ZenAppSetupException raise Exception("Module definition in configuration file is missing mandatory attribute 'exec'.") self.executable = params['exec'] self.name = params.get('name', self.executable) self.args = params.get('args', []) self.paralel = params.get('paralel', False) self.count = int(params.get('count', 0)) if self.paralel and self.count <= 1: # raise pyzenkit.baseapp.ZenAppSetupException raise Exception("Module '{}' is configured to run in paralel and the 'count' attribute is missing or below '2'.".format(self.name)) def __str__(self): return '{}'.format(self.name) def __repr__(self): return 'MentatModule(name={},executable={})'.format(self.name, self.executable)
[docs]class MentatCronjob(): """ Class representing Mentat cronjob module configuration for control utility. """ def __init__(self, params): if 'name' not in params: # raise pyzenkit.baseapp.ZenAppSetupException raise Exception("Cronjob definition in configuration file is missing mandatory attribute 'name'.") self.name = params['name'] def __str__(self): return '{}'.format(self.name) def __repr__(self): return 'MentatCronjob(name={})'.format(self.name)
[docs]def make_module_list(modules): """ *Helper function* """ result = collections.OrderedDict() for mod in modules: record = MentatModule(mod) result[record.name] = record return result
[docs]def make_cronjob_list(cronjobs): """ *Helper function* """ result = collections.OrderedDict() for mod in cronjobs: record = MentatCronjob(mod) result[record.name] = record return result
#-------------------------------------------------------------------------------
[docs]def analyze_process_ps(cmdline): """ Analyze given process command line received from ``ps`` utility. Example of returned structure:: { 'args': None, 'exec': 'mentat-storage.py', 'name': 'mentat-storage.py', 'paralel': False, 'pid': 9349, 'process': 'python3', 'psline': '9349 python3 python3 /usr/local/bin/mentat-storage.py' } :param str cmdline: Command line as received from ``ps`` utility). :return: Structure containing parsed process information. :rtype: dict """ match = REGEXP_MENTAT_PS.match(cmdline) if not match: return None if match.group(4) in ['mentat-controller.py']: return None record = { 'process': match.group(2), 'name': match.group(4), 'exec': match.group(4), 'args': match.group(5), 'pid': int(match.group(1)), 'psline': cmdline, 'paralel': False } # Attempt to parse command line arguments for selected data. if record['args']: # Attempt to parse custom name. match = re.search(r'--name(?: |=)([^\s]+)', record['args']) if match: record['name'] = match.group(1) # Attempt to detect paralel mode switch. match = re.search('--paralel', record['args']) if match: record['paralel'] = True return record
[docs]def analyze_process_list_ps(): """ Analyze all running processes using Linux`s ``ps`` utility. Processes are listed with following exact command:: /bin/ps axo pid,comm,args | grep mentat | grep -v grep Only processes that match the internal regular expression are selected. Example of returned structure:: { 'mentat-enricher.py': {9356: {'args': None, 'exec': 'mentat-enricher.py', 'name': 'mentat-enricher.py', 'paralel': False, 'pid': 9356, 'process': 'python3', 'psline': '9356 python3 python3 /usr/local/bin/mentat-enricher.py'}}, 'mentat-inspector.py': {9362: {'args': None, 'exec': 'mentat-inspector.py', 'name': 'mentat-inspector.py', 'paralel': False, 'pid': 9362, 'process': 'python3', 'psline': '9362 python3 python3 /usr/local/bin/mentat-inspector.py'}}, 'mentat-storage.py': {9349: {'args': None, 'exec': 'mentat-storage.py', 'name': 'mentat-storage.py', 'paralel': False, 'pid': 9349, 'process': 'python3', 'psline': '9349 python3 python3 /usr/local/bin/mentat-storage.py'}} } :return: Structured dictionary containing information about selected processess. :rtype: dict """ processes = collections.defaultdict(dict) with subprocess.Popen(['/bin/ps axo pid,comm,args | grep mentat | grep -v grep'], stdout = subprocess.PIPE, shell = True) as proc: for psline in proc.stdout: psline = psline.rstrip() psline = psline.decode('utf-8') record = analyze_process_ps(psline) if not record: continue processes[record['name']][record['pid']] = record return dict(processes)
[docs]def analyze_pid_file(pid_file, pid_file_path): """ Analyze given PID file. Example of returned structure:: { 'atime': datetime.datetime(2018, 1, 23, 8, 25, 16, 439742), 'file': 'mentat-storage.py.pid', 'mtime': datetime.datetime(2018, 1, 23, 8, 25, 12, 943727), 'name': 'mentat-storage.py', 'paralel': False, 'path': '/var/mentat/run/mentat-storage.py.pid', 'pid': 9349, 'size': 5 } :param str pid_file: Basename of the PID file. :param str pid_file_path: Full path to the PID file. :return: Structure containing parsed PID file information. :rtype: dict """ try: match = REGEXP_MENTAT_PIDF.match(pid_file) if not match: return None pid = None with open(pid_file_path, 'r') as pidf: pid = pidf.read() pid = int(pid) fsstat = os.stat(pid_file_path) return { 'name': match.group(1), 'file': pid_file, 'path': pid_file_path, 'pid': int(pid), 'paralel': True if match.group(2) else False, 'size': fsstat.st_size, 'atime': datetime.datetime.utcfromtimestamp(fsstat.st_atime), 'mtime': datetime.datetime.utcfromtimestamp(fsstat.st_mtime) } except: return None
[docs]def analyze_pid_files(pid_dir_path): """ Analyze all PID files in given run directory. Example of returned structure:: { 'mentat-enricher.py': {9356: {'atime': datetime.datetime(2018, 1, 23, 8, 25, 16, 439742), 'file': 'mentat-enricher.py.pid', 'mtime': datetime.datetime(2018, 1, 23, 8, 25, 14, 319733), 'name': 'mentat-enricher.py', 'paralel': False, 'path': '/var/mentat/run/mentat-enricher.py.pid', 'pid': 9356, 'size': 5}}, 'mentat-inspector.py': {9362: {'atime': datetime.datetime(2018, 1, 23, 8, 25, 16, 439742), 'file': 'mentat-inspector.py.pid', 'mtime': datetime.datetime(2018, 1, 23, 8, 25, 14, 947736), 'name': 'mentat-inspector.py', 'paralel': False, 'path': '/var/mentat/run/mentat-inspector.py.pid', 'pid': 9362, 'size': 5}}, 'mentat-storage.py': {9349: {'atime': datetime.datetime(2018, 1, 23, 8, 25, 16, 439742), 'file': 'mentat-storage.py.pid', 'mtime': datetime.datetime(2018, 1, 23, 8, 25, 12, 943727), 'name': 'mentat-storage.py', 'paralel': False, 'path': '/var/mentat/run/mentat-storage.py.pid', 'pid': 9349, 'size': 5}} } :param str pid_dir_path: path to directory containing PID files. :return: Structure containing parsed information for all PID files. :rtype: dict """ pid_files = collections.defaultdict(dict) all_files = os.listdir(pid_dir_path) for pid_file in all_files: pid_file_path = os.path.join(pid_dir_path, pid_file) if not os.path.isfile(pid_file_path): continue record = analyze_pid_file(pid_file, pid_file_path) if not record: continue pid_files[record['name']][record['pid']] = record return dict(pid_files)
[docs]def analyze_cron_file(cron_file, cron_file_path, cron_links): """ Analyze given cron file. Example of returned structure:: { 'atime': datetime.datetime(2018, 1, 21, 9, 13, 48, 34648), 'file': 'mentat-statistician-py.cron', 'link': None, 'mtime': datetime.datetime(2017, 7, 19, 10, 25, 30), 'name': 'mentat-statistician-py', 'path': '/etc/mentat/cron/mentat-statistician-py.cron', 'size': 429 } :param str cron_file: Basename of the cron file. :param str cron_file_path: Full path to the cron file. :param dict cron_links: Dictionary containing existing links in cron directory. :return: Structure containing parsed cron file information. :rtype: dict """ try: match = REGEXP_MENTAT_CRONF.match(cron_file) if not match: return None fsstat = os.stat(cron_file_path) return { 'name': match.group(1), 'file': cron_file, 'path': cron_file_path, 'size': fsstat.st_size, 'atime': datetime.datetime.utcfromtimestamp(fsstat.st_atime), 'mtime': datetime.datetime.utcfromtimestamp(fsstat.st_mtime), 'link': cron_links.get(cron_file_path, None) } except: return None
[docs]def analyze_cron_files(cfg_dir_path, cron_dir_path): """ Analyze all cron files in config and cron directory. Example of returned structure:: { 'mentat-precache-py': {'atime': datetime.datetime(2018, 1, 21, 9, 13, 45), 'file': 'mentat-precache-py.cron', 'link': None, 'mtime': datetime.datetime(2017, 9, 1, 11, 10, 17), 'name': 'mentat-precache-py', 'path': '/etc/mentat/cron/mentat-precache-py.cron', 'size': 417}, 'mentat-statistician-py': {'atime': datetime.datetime(2018, 1, 21, 9, 13, 48, 34648), 'file': 'mentat-statistician-py.cron', 'link': None, 'mtime': datetime.datetime(2017, 7, 19, 10, 25, 30), 'name': 'mentat-statistician-py.cron', 'path': '/etc/mentat/cron/mentat-statistician-py.cron', 'size': 429} } :param str cfg_dir_path: Path to configuration directory containing cron scripts. :param str cron_dir_path: Path to system cron directory. :return: Structure containing parsed cron file information. :rtype: dict """ cron_files = {} cron_links = {} crn_all_files = os.listdir(cron_dir_path) for cron_file in crn_all_files: cron_file_path = os.path.join(cron_dir_path, cron_file) if not os.path.islink(cron_file_path): continue cron_links[os.readlink(cron_file_path)] = cron_file_path cfg_all_files = os.listdir(cfg_dir_path) for cron_file in cfg_all_files: cron_file_path = os.path.join(cfg_dir_path, cron_file) if not os.path.isfile(cron_file_path): continue record = analyze_cron_file(cron_file, cron_file_path, cron_links) if not record: continue cron_files[record['name']] = record return cron_files
[docs]def analyze_log_file(log_file, log_file_path): """ Analyze given log file. Example of returned structure:: { 'atime': datetime.datetime(2018, 1, 22, 15, 11, 15, 562358), 'file': 'mentat-storage.py.log', 'mtime': datetime.datetime(2018, 1, 23, 9, 8, 43, 403734), 'name': 'mentat-storage.py', 'path': '/var/mentat/log/mentat-storage.py.log', 'size': 5831678 } :param str log_file: Basename of the log file. :param str log_file_path: Full path to the log file. :return: Structure containing parsed log file information. :rtype: dict """ try: match = REGEXP_MENTAT_LOGF.match(log_file) if not match: return None fsstat = os.stat(log_file_path) return { 'name': match.group(1), 'file': log_file, 'path': log_file_path, 'size': fsstat.st_size, 'atime': datetime.datetime.utcfromtimestamp(fsstat.st_atime), 'mtime': datetime.datetime.utcfromtimestamp(fsstat.st_mtime) } except: return None
[docs]def analyze_log_files(log_dir_path): """ Analyze all PID files in run directory. Example of returned structure:: { 'mentat-enricher.py': {'atime': datetime.datetime(2018, 1, 9, 12, 25, 26, 338720), 'file': 'mentat-enricher.py.log', 'mtime': datetime.datetime(2018, 1, 23, 9, 8, 38, 163710), 'name': 'mentat-enricher.py', 'path': '/var/mentat/log/mentat-enricher.py.log', 'size': 6057586}, 'mentat-inspector.py': {'atime': datetime.datetime(2018, 1, 18, 17, 5, 22, 734164), 'file': 'mentat-inspector.py.log', 'mtime': datetime.datetime(2018, 1, 23, 9, 8, 35, 291697), 'name': 'mentat-inspector.py', 'path': '/var/mentat/log/mentat-inspector.py.log', 'size': 8156449}, 'mentat-storage.py': {'atime': datetime.datetime(2018, 1, 22, 15, 11, 15, 562358), 'file': 'mentat-storage.py.log', 'mtime': datetime.datetime(2018, 1, 23, 9, 8, 43, 403734), 'name': 'mentat-storage.py', 'path': '/var/mentat/log/mentat-storage.py.log', 'size': 5831678} } :param str log_dir_path: Path to directory containing log files. :return: Structure containing parsed log file information. :rtype: dict """ log_files = {} all_files = os.listdir(log_dir_path) for log_file in all_files: log_file_path = os.path.join(log_dir_path, log_file) if not os.path.isfile(log_file_path): continue record = analyze_log_file(log_file, log_file_path) if not record: continue log_files[record['name']] = record return log_files
[docs]def analyze_runlog_file(runlog_file, runlog_file_path): """ Analyze given runlog file. Example of returned structure:: { 'atime': datetime.datetime(2018, 1, 22, 16, 33, 56, 766819), 'data': {...}, 'file': '201801221611.runlog', 'mtime': datetime.datetime(2018, 1, 22, 15, 30, 0, 427902), 'name': '201801221611', 'path': '/var/mentat/run/mentat-storage.py/201801221611.runlog', 'size': 1209 } :param str runlog_file: Basename of the runlog file. :param str runlog_file_path: Full path to the runlog file. :return: Structure containing parsed runlog file information. :rtype: dict """ try: match = REGEXP_MENTAT_RLOGF.match(runlog_file) if not match: return None fsstat = os.stat(runlog_file_path) return { 'name': match.group(1), 'file': runlog_file, 'path': runlog_file_path, 'size': fsstat.st_size, 'atime': datetime.datetime.utcfromtimestamp(fsstat.st_atime), 'mtime': datetime.datetime.utcfromtimestamp(fsstat.st_mtime), 'data': pyzenkit.jsonconf.json_load(runlog_file_path) } except: return None
[docs]def analyze_runlog_files(run_dir_path, limit = None): """ Analyze all runlog files in run directory. :param str run_dir_path: Path to directory containing run files. :return: Structure containing parsed runlog file information. :rtype: dict """ runlog_files = collections.defaultdict(dict) all_files = os.listdir(run_dir_path) for runlog_dir in all_files: runlog_dir_path = os.path.join(run_dir_path, runlog_dir) if not os.path.isdir(runlog_dir_path): continue all_rl_files = sorted(os.listdir(runlog_dir_path), reverse = True) file_counter = 0 for runlog_file in all_rl_files: runlog_file_path = os.path.join(runlog_dir_path, runlog_file) if not os.path.isfile(runlog_file_path): continue record = analyze_runlog_file(runlog_file, runlog_file_path) if not record: continue file_counter += 1 if limit and file_counter > limit: break runlog_files[runlog_dir][record['name']] = record return dict(runlog_files)
[docs]def analyze_cache_file(cache_file, cache_file_path): """ Analyze given cache file. Example of returned structure:: { 'atime': datetime.datetime(2018, 1, 22, 15, 11, 15, 562358), 'file': 'itemset-stat-detectors.json', 'mtime': datetime.datetime(2018, 1, 23, 9, 8, 43, 403734), 'name': 'itemset-stat-detectors', 'path': '/var/mentat/cache/itemset-stat-detectors.json', 'size': 5831678 } :param str cache_file: Basename of the cache file. :param str cache_file_path: Full path to the cache file. :return: Structure containing parsed cache file information. :rtype: dict """ try: match = REGEXP_MENTAT_CACHEF.match(cache_file) if not match: return None fsstat = os.stat(cache_file_path) return { 'name': match.group(1), 'file': cache_file, 'path': cache_file_path, 'size': fsstat.st_size, 'atime': datetime.datetime.utcfromtimestamp(fsstat.st_atime), 'mtime': datetime.datetime.utcfromtimestamp(fsstat.st_mtime), 'data': pyzenkit.jsonconf.json_load(cache_file_path) } except: return None
[docs]def analyze_cache_files(cache_dir_path): """ Analyze all cache files in cache directory. Example of returned structure:: { 'itemset-stat-detectors': {'atime': datetime.datetime(2018, 1, 9, 12, 25, 26, 338720), 'file': 'itemset-stat-detectors.json', 'mtime': datetime.datetime(2018, 1, 23, 9, 8, 38, 163710), 'name': 'mentat-enricher.py', 'path': '/var/mentat/cache/itemset-stat-detectors.json', 'size': 6057586, 'data': {...}} } :param str cache_dir_path: Path to directory containing cache files. :return: Structure containing parsed cache file information. :rtype: dict """ cache_files = {} all_files = os.listdir(cache_dir_path) for cache_file in all_files: cache_file_path = os.path.join(cache_dir_path, cache_file) if not os.path.isfile(cache_file_path): continue record = analyze_cache_file(cache_file, cache_file_path) if not record: continue cache_files[record['name']] = record return cache_files
#-------------------------------------------------------------------------------
[docs]def module_status(mod_data, pidf_data, proc_data, overall_status): """ Analyze status of given module. """ # Check the consistency of running processes and pid files. consistent = True if pidf_data: for pid in pidf_data.keys(): if not proc_data or not pid in proc_data: overall_status['messages']['error'].append(( "Process '{}' is dead, but pid file '{}' exists".format( pid, pidf_data[pid]['path'] ), 'pidfile', pidf_data[pid]['path'] )) consistent = False if proc_data: for pid in proc_data.keys(): if not pidf_data or not pid in pidf_data: overall_status['messages']['error'].append(( "Process '{}':'{}' is running, but pid file is missing".format( pid, proc_data[pid]['name'] ), 'process', pid )) consistent = False # PID file exists and process is running. if pidf_data and proc_data: # Check the number of running processes if mod_data.paralel: if mod_data.count > len(proc_data.keys()): return ( STATUS_RT_RUNNING_FEWER, "There are fewer instances running than required ({}:{})".format(mod_data.count, len(proc_data.keys())) ) if mod_data.count < len(proc_data.keys()): return ( STATUS_RT_RUNNING_MORE, "There are more instances running than required ({}:{})".format(mod_data.count, len(proc_data.keys())) ) # Process is running or service is OK if consistent: return ( STATUS_RT_RUNNING_OK, "Process is running or service is OK ({})".format(len(proc_data.keys())) ) return ( STATUS_RT_RUNNING_INCONSISTENT, "Process is running in inconsistent state" ) # PID file(s) exist(s) and any process is not running if pidf_data: if mod_data.paralel: return ( STATUS_RT_DEAD_PF_EXISTS, "Processes are dead, but some pid files exist" ) return ( STATUS_RT_DEAD_PF_EXISTS, "Process is dead, but pid file exist" ) # PID file(s) do(es) not exist, but some process is running if proc_data: if mod_data.paralel: return ( STATUS_RT_RUNNING_PF_MISSING, "Processes are running, but pid files are missing" ) return ( STATUS_RT_RUNNING_PF_MISSING, "Process is running, but pid file is missing" ) # Process is not running return ( STATUS_RT_NOT_RUNNING, "Process is not running or service is stopped" )
# TODO: Process is dead and /var/lock lock file exists #return STATUS_RT_DEAD_LOCK_EXISTS # TODO: Process or service status is unknown #return STATUS_RT_UNKNOWN
[docs]def cronjob_status(cronj_data, cronf_data, overall_status): """ Analyze status of given cron module. """ if cronf_data and cronf_data['link']: return ( STATUS_CJ_ENABLED, "Cronjob is enabled" ) return ( STATUS_CJ_DISABLED, "Cronjob is disabled" )
[docs]def system_status(modules, cronjobs, cfg_dir_path, cron_dir_path, log_dir_path, run_dir_path): """ Analyze status of all modules and cronjobs and detect additional problems. """ status = {} status['dt'] = datetime.datetime.utcnow() status['modules'] = {} status['cronjobs'] = {} status['messages'] = {'info': [], 'notice': [], 'warning': [], 'error': []} status['processes'] = analyze_process_list_ps() status['pid_files'] = analyze_pid_files(run_dir_path) status['log_files'] = analyze_log_files(log_dir_path) status['runlog_files'] = analyze_runlog_files(run_dir_path, limit = 1) status['cron_files'] = analyze_cron_files(os.path.join(cfg_dir_path, 'cron'), cron_dir_path) status['resultm'] = None status['resultc'] = None status['result'] = None # A) Analyze status of configured modules. for mname, mdata in modules.items(): status['modules'][mname] = module_status( mdata, status['pid_files'].get(mname, None), status['processes'].get(mname, None), status ) # B) Analyze status of configured cronjob modules. for mname, mdata in cronjobs.items(): status['cronjobs'][mname] = cronjob_status( mdata, status['cron_files'].get(mname, None), status ) # ----- _analyze_unconfigured_pid_files(status, modules) _analyze_unconfigured_modules(status, modules) _analyze_unconfigured_cronjobs(status, cronjobs) # ----- # Determine overall status of real-time processing modules. for mname, mstatus in status['modules'].items(): if not status['resultm']: status['resultm'] = (mstatus[0], OVERALL_MODULE_STATUS_MESSAGES[mstatus[0]]) elif status['resultm'][0] != mstatus[0]: status['resultm'] = (STATUS_RT_UNKNOWN, OVERALL_MODULE_STATUS_MESSAGES[STATUS_RT_UNKNOWN]) if not status['resultm']: status['resultm'] = (STATUS_RT_NOT_RUNNING, OVERALL_MODULE_STATUS_MESSAGES[STATUS_RT_NOT_RUNNING]) # Determine overall status of cronjob modules. for mname, mstatus in status['cronjobs'].items(): if not status['resultc']: status['resultc'] = (mstatus[0], OVERALL_CRONJOB_STATUS_MESSAGES[mstatus[0]]) elif status['resultc'][0] != mstatus[0]: status['resultc'] = (STATUS_CJ_UNKNOWN, OVERALL_CRONJOB_STATUS_MESSAGES[STATUS_CJ_UNKNOWN]) if not status['resultc']: status['resultc'] = (STATUS_CJ_DISABLED, OVERALL_CRONJOB_STATUS_MESSAGES[STATUS_CJ_DISABLED]) # Determine overall status of the whole system. if status['resultm'][0] != status['resultc'][0]: status['result'] = (STATUS_UNKNOWN, OVERALL_STATUS_MESSAGES[STATUS_UNKNOWN]) else: status['result'] = (status['resultm'][0], OVERALL_STATUS_MESSAGES[status['resultm'][0]]) return status
[docs]def analyze_versions(): """ Analyze versions of various relevant dependencies (like PostgreSQL, ...). """ result = {} out = subprocess.check_output(['psql','-V']) result['postgresql'] = out.decode('utf-8').strip() return result
#------------------------------------------------------------------------------- def _analyze_unconfigured_pid_files(status, modules): """ Analyze list of all PID files and find any that do not belong to known modules. """ for mname in sorted(status['pid_files'].keys()): if mname in modules: continue for pid in sorted(status['pid_files'][mname].keys()): status['messages']['info'].append(( "Unknown pid file '{}' for PID '{}'".format( status['pid_files'][mname][pid]['path'], pid ), 'pidfile', status['pid_files'][mname][pid]['path'] )) def _analyze_unconfigured_modules(status, modules): """ Analyze list of all processes files and find any that do not belong to known modules. """ for mname in sorted(status['processes'].keys()): if mname in modules: continue for pid in sorted(status['processes'][mname].keys()): status['messages']['info'].append(( "Unknown process '{}' with PID '{}'".format( status['processes'][mname][pid]['exec'], pid ), 'pidfile', pid )) def _analyze_unconfigured_cronjobs(status, cronjobs): """ Analyze list of all cron files and find any that do not belong to known cronjob modules. """ for mname in sorted(status['cron_files'].keys()): if mname in cronjobs: continue if status['cron_files'][mname]['link']: status['messages']['info'].append(( "Unconfigured cron file '{}' for cron module '{}'".format( status['cron_files'][mname]['link'], status['cron_files'][mname]['path'] ), 'cronfile', status['cron_files'][mname]['path'] ))