#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# This file is part of Mentat system (https://mentat.cesnet.cz/).
#
# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
"""
Mentat system management and inspection library.
"""
__author__ = "Jan Mach <jan.mach@cesnet.cz>"
__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
import os
import re
import datetime
import collections
import subprocess
import pyzenkit.jsonconf
#
# Real-time module status codes.
#
STATUS_RT_RUNNING_OK = 0 # Module is running or service is OK
STATUS_RT_RUNNING_PF_MISSING = 1 # Module is running, but pid file is missing
STATUS_RT_RUNNING_PID_MISSMATCH = 2 # Module is running, but pid file contains different PID
STATUS_RT_RUNNING_INCONSISTENT = 10 # Module is running, but pid files are inconsistent
STATUS_RT_RUNNING_FEWER = 11 # Module is running in fewer that required instances
STATUS_RT_RUNNING_MORE = 12 # Module is running in more that required instances
STATUS_RT_DEAD_PF_EXISTS = 3 # Module is dead and pid file exists
STATUS_RT_DEAD_LOCK_EXISTS = 4 # Module is dead and lock file exists
STATUS_RT_NOT_RUNNING = 5 # Module is not running
STATUS_RT_UNKNOWN = 6 # Module or service status is unknown
# Overall module status messages
OVERALL_MODULE_STATUS_MESSAGES = {
STATUS_RT_RUNNING_OK: "All modules are running OK",
STATUS_RT_RUNNING_PF_MISSING: "All modules are running, but all pid files are missing",
STATUS_RT_RUNNING_PID_MISSMATCH: "All modules are running, but all pid files contain different PID",
STATUS_RT_RUNNING_INCONSISTENT: "All modules are running, but all pid files are inconsistent",
STATUS_RT_RUNNING_FEWER: "All modules are running in fewer that required instances",
STATUS_RT_RUNNING_MORE: "All modules are running in more that required instances",
STATUS_RT_DEAD_PF_EXISTS: "All modules are dead and all pid files exist",
STATUS_RT_DEAD_LOCK_EXISTS: "All modules are dead and all lock files exist",
STATUS_RT_NOT_RUNNING: "All modules are not running",
STATUS_RT_UNKNOWN: "Some modules are in different state, overall system state is unknown",
}
#
# Cronjob module status codes.
#
STATUS_CJ_ENABLED = 0 # Cronjob is enabled
STATUS_CJ_DISABLED = 5 # Cronjob is not running
STATUS_CJ_UNKNOWN = 6 # Cronjob status is unknown
# Overall cronjob status messages
OVERALL_CRONJOB_STATUS_MESSAGES = {
STATUS_CJ_ENABLED: "All cronjobs are enabled",
STATUS_CJ_DISABLED: "All cronjobs are disabled",
STATUS_CJ_UNKNOWN: "Some cronjobs are in different state, overall system state is unknown",
}
#
# Cronjob module status codes.
#
STATUS_ONLINE = 0 # The whole system is online
STATUS_OFFLINE = 5 # The whole system is offline
STATUS_UNKNOWN = 6 # System overall status is unknown
# Overall cronjob status messages
OVERALL_STATUS_MESSAGES = {
STATUS_ONLINE: "All modules are running OK and all cronjobs are enabled",
STATUS_OFFLINE: "All modules are NOT running and all cronjobs are disabled",
STATUS_UNKNOWN: "Some modules and cronjobs are in different state, overall system state is unknown",
}
REGEXP_MENTAT_PS = re.compile(r'\s*(\d+)\s+([^\s]+)\s+([^\s]*(?:python(?:\d+(?:\.\d+)?)?))\s+[^\s]+(mentat-[^\s]+)(?:\s+(.*))?')
"""Regular expression for selecting Mentat related processes."""
REGEXP_MENTAT_PIDF = re.compile(r'(.+?)(?:\.([0-9a-fA-F]+))?\.pid$')
"""Regular expression for selecting Mentat related PID files."""
REGEXP_MENTAT_CRONF = re.compile(r'(.+)\.cron$')
"""Regular expression for selecting Mentat related log files."""
REGEXP_MENTAT_LOGF = re.compile(r'(.+)\.log$')
"""Regular expression for selecting Mentat related log files."""
REGEXP_MENTAT_RLOGF = re.compile(r'(.+)(?:\.([0-9a-fA-F]+))?\.runlog$')
"""Regular expression for selecting Mentat related runlog files."""
REGEXP_MENTAT_CACHEF = re.compile(r'(.+)\.json$')
"""Regular expression for selecting Mentat related cache files."""
[docs]class MentatModule():
"""
Class representing Mentat real-time module configuration for control utility.
"""
def __init__(self, params):
if 'exec' not in params:
# raise pyzenkit.baseapp.ZenAppSetupException
raise Exception("Module definition in configuration file is missing mandatory attribute 'exec'.")
self.executable = params['exec']
self.name = params.get('name', self.executable)
self.args = params.get('args', [])
self.paralel = params.get('paralel', False)
self.count = int(params.get('count', 0))
if self.paralel and self.count <= 1:
# raise pyzenkit.baseapp.ZenAppSetupException
raise Exception("Module '{}' is configured to run in paralel and the 'count' attribute is missing or below '2'.".format(self.name))
def __str__(self):
return '{}'.format(self.name)
def __repr__(self):
return 'MentatModule(name={},executable={})'.format(self.name, self.executable)
[docs]class MentatCronjob():
"""
Class representing Mentat cronjob module configuration for control utility.
"""
def __init__(self, params):
if 'name' not in params:
# raise pyzenkit.baseapp.ZenAppSetupException
raise Exception("Cronjob definition in configuration file is missing mandatory attribute 'name'.")
self.name = params['name']
def __str__(self):
return '{}'.format(self.name)
def __repr__(self):
return 'MentatCronjob(name={})'.format(self.name)
[docs]def make_module_list(modules):
"""
*Helper function*
"""
result = collections.OrderedDict()
for mod in modules:
record = MentatModule(mod)
result[record.name] = record
return result
[docs]def make_cronjob_list(cronjobs):
"""
*Helper function*
"""
result = collections.OrderedDict()
for mod in cronjobs:
record = MentatCronjob(mod)
result[record.name] = record
return result
#-------------------------------------------------------------------------------
[docs]def analyze_process_ps(cmdline):
"""
Analyze given process command line received from ``ps`` utility.
Example of returned structure::
{
'args': None,
'exec': 'mentat-storage.py',
'name': 'mentat-storage.py',
'paralel': False,
'pid': 9349,
'process': 'python3',
'psline': '9349 python3 python3 /usr/local/bin/mentat-storage.py'
}
:param str cmdline: Command line as received from ``ps`` utility).
:return: Structure containing parsed process information.
:rtype: dict
"""
match = REGEXP_MENTAT_PS.match(cmdline)
if not match:
return None
if match.group(4) in ['mentat-controller.py']:
return None
record = {
'process': match.group(2),
'name': match.group(4),
'exec': match.group(4),
'args': match.group(5),
'pid': int(match.group(1)),
'psline': cmdline,
'paralel': False
}
# Attempt to parse command line arguments for selected data.
if record['args']:
# Attempt to parse custom name.
match = re.search(r'--name(?: |=)([^\s]+)', record['args'])
if match:
record['name'] = match.group(1)
# Attempt to detect paralel mode switch.
match = re.search('--paralel', record['args'])
if match:
record['paralel'] = True
return record
[docs]def analyze_process_list_ps():
"""
Analyze all running processes using Linux`s ``ps`` utility.
Processes are listed with following exact command::
/bin/ps axo pid,comm,args | grep mentat | grep -v grep
Only processes that match the internal regular expression are selected.
Example of returned structure::
{
'mentat-enricher.py': {9356: {'args': None,
'exec': 'mentat-enricher.py',
'name': 'mentat-enricher.py',
'paralel': False,
'pid': 9356,
'process': 'python3',
'psline': '9356 python3 python3 /usr/local/bin/mentat-enricher.py'}},
'mentat-inspector.py': {9362: {'args': None,
'exec': 'mentat-inspector.py',
'name': 'mentat-inspector.py',
'paralel': False,
'pid': 9362,
'process': 'python3',
'psline': '9362 python3 python3 /usr/local/bin/mentat-inspector.py'}},
'mentat-storage.py': {9349: {'args': None,
'exec': 'mentat-storage.py',
'name': 'mentat-storage.py',
'paralel': False,
'pid': 9349,
'process': 'python3',
'psline': '9349 python3 python3 /usr/local/bin/mentat-storage.py'}}
}
:return: Structured dictionary containing information about selected processess.
:rtype: dict
"""
processes = collections.defaultdict(dict)
with subprocess.Popen(['/bin/ps axo pid,comm,args | grep mentat | grep -v grep'], stdout = subprocess.PIPE, shell = True) as proc:
for psline in proc.stdout:
psline = psline.rstrip()
psline = psline.decode('utf-8')
record = analyze_process_ps(psline)
if not record:
continue
processes[record['name']][record['pid']] = record
return dict(processes)
[docs]def analyze_pid_file(pid_file, pid_file_path):
"""
Analyze given PID file.
Example of returned structure::
{
'atime': datetime.datetime(2018, 1, 23, 8, 25, 16, 439742),
'file': 'mentat-storage.py.pid',
'mtime': datetime.datetime(2018, 1, 23, 8, 25, 12, 943727),
'name': 'mentat-storage.py',
'paralel': False,
'path': '/var/mentat/run/mentat-storage.py.pid',
'pid': 9349,
'size': 5
}
:param str pid_file: Basename of the PID file.
:param str pid_file_path: Full path to the PID file.
:return: Structure containing parsed PID file information.
:rtype: dict
"""
try:
match = REGEXP_MENTAT_PIDF.match(pid_file)
if not match:
return None
pid = None
with open(pid_file_path, 'r', encoding="utf8") as pidf:
pid = pidf.read()
pid = int(pid)
fsstat = os.stat(pid_file_path)
return {
'name': match.group(1),
'file': pid_file,
'path': pid_file_path,
'pid': int(pid),
'paralel': bool(match.group(2)),
'size': fsstat.st_size,
'atime': datetime.datetime.utcfromtimestamp(fsstat.st_atime),
'mtime': datetime.datetime.utcfromtimestamp(fsstat.st_mtime)
}
except Exception:
return None
[docs]def analyze_pid_files(pid_dir_path):
"""
Analyze all PID files in given run directory.
Example of returned structure::
{
'mentat-enricher.py': {9356: {'atime': datetime.datetime(2018, 1, 23, 8, 25, 16, 439742),
'file': 'mentat-enricher.py.pid',
'mtime': datetime.datetime(2018, 1, 23, 8, 25, 14, 319733),
'name': 'mentat-enricher.py',
'paralel': False,
'path': '/var/mentat/run/mentat-enricher.py.pid',
'pid': 9356,
'size': 5}},
'mentat-inspector.py': {9362: {'atime': datetime.datetime(2018, 1, 23, 8, 25, 16, 439742),
'file': 'mentat-inspector.py.pid',
'mtime': datetime.datetime(2018, 1, 23, 8, 25, 14, 947736),
'name': 'mentat-inspector.py',
'paralel': False,
'path': '/var/mentat/run/mentat-inspector.py.pid',
'pid': 9362,
'size': 5}},
'mentat-storage.py': {9349: {'atime': datetime.datetime(2018, 1, 23, 8, 25, 16, 439742),
'file': 'mentat-storage.py.pid',
'mtime': datetime.datetime(2018, 1, 23, 8, 25, 12, 943727),
'name': 'mentat-storage.py',
'paralel': False,
'path': '/var/mentat/run/mentat-storage.py.pid',
'pid': 9349,
'size': 5}}
}
:param str pid_dir_path: path to directory containing PID files.
:return: Structure containing parsed information for all PID files.
:rtype: dict
"""
pid_files = collections.defaultdict(dict)
all_files = os.listdir(pid_dir_path)
for pid_file in all_files:
pid_file_path = os.path.join(pid_dir_path, pid_file)
if not os.path.isfile(pid_file_path):
continue
record = analyze_pid_file(pid_file, pid_file_path)
if not record:
continue
pid_files[record['name']][record['pid']] = record
return dict(pid_files)
[docs]def analyze_cron_file(cron_file, cron_file_path, cron_links):
"""
Analyze given cron file.
Example of returned structure::
{
'atime': datetime.datetime(2018, 1, 21, 9, 13, 48, 34648),
'file': 'mentat-statistician-py.cron',
'link': None,
'mtime': datetime.datetime(2017, 7, 19, 10, 25, 30),
'name': 'mentat-statistician-py',
'path': '/etc/mentat/cron/mentat-statistician-py.cron',
'size': 429
}
:param str cron_file: Basename of the cron file.
:param str cron_file_path: Full path to the cron file.
:param dict cron_links: Dictionary containing existing links in cron directory.
:return: Structure containing parsed cron file information.
:rtype: dict
"""
try:
match = REGEXP_MENTAT_CRONF.match(cron_file)
if not match:
return None
fsstat = os.stat(cron_file_path)
return {
'name': match.group(1),
'file': cron_file,
'path': cron_file_path,
'size': fsstat.st_size,
'atime': datetime.datetime.utcfromtimestamp(fsstat.st_atime),
'mtime': datetime.datetime.utcfromtimestamp(fsstat.st_mtime),
'link': cron_links.get(cron_file_path, None)
}
except Exception:
return None
[docs]def analyze_cron_files(cfg_dir_path, cron_dir_path):
"""
Analyze all cron files in config and cron directory.
Example of returned structure::
{
'mentat-precache-py': {'atime': datetime.datetime(2018, 1, 21, 9, 13, 45),
'file': 'mentat-precache-py.cron',
'link': None,
'mtime': datetime.datetime(2017, 9, 1, 11, 10, 17),
'name': 'mentat-precache-py',
'path': '/etc/mentat/cron/mentat-precache-py.cron',
'size': 417},
'mentat-statistician-py': {'atime': datetime.datetime(2018, 1, 21, 9, 13, 48, 34648),
'file': 'mentat-statistician-py.cron',
'link': None,
'mtime': datetime.datetime(2017, 7, 19, 10, 25, 30),
'name': 'mentat-statistician-py.cron',
'path': '/etc/mentat/cron/mentat-statistician-py.cron',
'size': 429}
}
:param str cfg_dir_path: Path to configuration directory containing cron scripts.
:param str cron_dir_path: Path to system cron directory.
:return: Structure containing parsed cron file information.
:rtype: dict
"""
cron_files = {}
cron_links = {}
crn_all_files = os.listdir(cron_dir_path)
for cron_file in crn_all_files:
cron_file_path = os.path.join(cron_dir_path, cron_file)
if not os.path.islink(cron_file_path):
continue
cron_links[os.readlink(cron_file_path)] = cron_file_path
cfg_all_files = os.listdir(cfg_dir_path)
for cron_file in cfg_all_files:
cron_file_path = os.path.join(cfg_dir_path, cron_file)
if not os.path.isfile(cron_file_path):
continue
record = analyze_cron_file(cron_file, cron_file_path, cron_links)
if not record:
continue
cron_files[record['name']] = record
return cron_files
[docs]def analyze_log_file(log_file, log_file_path):
"""
Analyze given log file.
Example of returned structure::
{
'atime': datetime.datetime(2018, 1, 22, 15, 11, 15, 562358),
'file': 'mentat-storage.py.log',
'mtime': datetime.datetime(2018, 1, 23, 9, 8, 43, 403734),
'name': 'mentat-storage.py',
'path': '/var/mentat/log/mentat-storage.py.log',
'size': 5831678
}
:param str log_file: Basename of the log file.
:param str log_file_path: Full path to the log file.
:return: Structure containing parsed log file information.
:rtype: dict
"""
try:
match = REGEXP_MENTAT_LOGF.match(log_file)
if not match:
return None
fsstat = os.stat(log_file_path)
return {
'name': match.group(1),
'file': log_file,
'path': log_file_path,
'size': fsstat.st_size,
'atime': datetime.datetime.utcfromtimestamp(fsstat.st_atime),
'mtime': datetime.datetime.utcfromtimestamp(fsstat.st_mtime)
}
except Exception:
return None
[docs]def analyze_log_files(log_dir_path):
"""
Analyze all PID files in run directory.
Example of returned structure::
{
'mentat-enricher.py': {'atime': datetime.datetime(2018, 1, 9, 12, 25, 26, 338720),
'file': 'mentat-enricher.py.log',
'mtime': datetime.datetime(2018, 1, 23, 9, 8, 38, 163710),
'name': 'mentat-enricher.py',
'path': '/var/mentat/log/mentat-enricher.py.log',
'size': 6057586},
'mentat-inspector.py': {'atime': datetime.datetime(2018, 1, 18, 17, 5, 22, 734164),
'file': 'mentat-inspector.py.log',
'mtime': datetime.datetime(2018, 1, 23, 9, 8, 35, 291697),
'name': 'mentat-inspector.py',
'path': '/var/mentat/log/mentat-inspector.py.log',
'size': 8156449},
'mentat-storage.py': {'atime': datetime.datetime(2018, 1, 22, 15, 11, 15, 562358),
'file': 'mentat-storage.py.log',
'mtime': datetime.datetime(2018, 1, 23, 9, 8, 43, 403734),
'name': 'mentat-storage.py',
'path': '/var/mentat/log/mentat-storage.py.log',
'size': 5831678}
}
:param str log_dir_path: Path to directory containing log files.
:return: Structure containing parsed log file information.
:rtype: dict
"""
log_files = {}
all_files = os.listdir(log_dir_path)
for log_file in all_files:
log_file_path = os.path.join(log_dir_path, log_file)
if not os.path.isfile(log_file_path):
continue
record = analyze_log_file(log_file, log_file_path)
if not record:
continue
log_files[record['name']] = record
return log_files
[docs]def analyze_runlog_file(runlog_file, runlog_file_path):
"""
Analyze given runlog file.
Example of returned structure::
{
'atime': datetime.datetime(2018, 1, 22, 16, 33, 56, 766819),
'data': {...},
'file': '201801221611.runlog',
'mtime': datetime.datetime(2018, 1, 22, 15, 30, 0, 427902),
'name': '201801221611',
'path': '/var/mentat/run/mentat-storage.py/201801221611.runlog',
'size': 1209
}
:param str runlog_file: Basename of the runlog file.
:param str runlog_file_path: Full path to the runlog file.
:return: Structure containing parsed runlog file information.
:rtype: dict
"""
try:
match = REGEXP_MENTAT_RLOGF.match(runlog_file)
if not match:
return None
fsstat = os.stat(runlog_file_path)
return {
'name': match.group(1),
'file': runlog_file,
'path': runlog_file_path,
'size': fsstat.st_size,
'atime': datetime.datetime.utcfromtimestamp(fsstat.st_atime),
'mtime': datetime.datetime.utcfromtimestamp(fsstat.st_mtime),
'data': pyzenkit.jsonconf.json_load(runlog_file_path)
}
except Exception:
return None
[docs]def analyze_runlog_files(run_dir_path, limit = None):
"""
Analyze all runlog files in run directory.
:param str run_dir_path: Path to directory containing run files.
:return: Structure containing parsed runlog file information.
:rtype: dict
"""
runlog_files = collections.defaultdict(dict)
all_files = os.listdir(run_dir_path)
for runlog_dir in all_files:
runlog_dir_path = os.path.join(run_dir_path, runlog_dir)
if not os.path.isdir(runlog_dir_path):
continue
all_rl_files = sorted(os.listdir(runlog_dir_path), reverse = True)
file_counter = 0
for runlog_file in all_rl_files:
runlog_file_path = os.path.join(runlog_dir_path, runlog_file)
if not os.path.isfile(runlog_file_path):
continue
record = analyze_runlog_file(runlog_file, runlog_file_path)
if not record:
continue
file_counter += 1
if limit and file_counter > limit:
break
runlog_files[runlog_dir][record['name']] = record
return dict(runlog_files)
[docs]def analyze_cache_file(cache_file, cache_file_path):
"""
Analyze given cache file.
Example of returned structure::
{
'atime': datetime.datetime(2018, 1, 22, 15, 11, 15, 562358),
'file': 'itemset-stat-detectors.json',
'mtime': datetime.datetime(2018, 1, 23, 9, 8, 43, 403734),
'name': 'itemset-stat-detectors',
'path': '/var/mentat/cache/itemset-stat-detectors.json',
'size': 5831678
}
:param str cache_file: Basename of the cache file.
:param str cache_file_path: Full path to the cache file.
:return: Structure containing parsed cache file information.
:rtype: dict
"""
try:
match = REGEXP_MENTAT_CACHEF.match(cache_file)
if not match:
return None
fsstat = os.stat(cache_file_path)
return {
'name': match.group(1),
'file': cache_file,
'path': cache_file_path,
'size': fsstat.st_size,
'atime': datetime.datetime.utcfromtimestamp(fsstat.st_atime),
'mtime': datetime.datetime.utcfromtimestamp(fsstat.st_mtime),
'data': pyzenkit.jsonconf.json_load(cache_file_path)
}
except Exception:
return None
[docs]def analyze_cache_files(cache_dir_path):
"""
Analyze all cache files in cache directory.
Example of returned structure::
{
'itemset-stat-detectors': {'atime': datetime.datetime(2018, 1, 9, 12, 25, 26, 338720),
'file': 'itemset-stat-detectors.json',
'mtime': datetime.datetime(2018, 1, 23, 9, 8, 38, 163710),
'name': 'mentat-enricher.py',
'path': '/var/mentat/cache/itemset-stat-detectors.json',
'size': 6057586,
'data': {...}}
}
:param str cache_dir_path: Path to directory containing cache files.
:return: Structure containing parsed cache file information.
:rtype: dict
"""
cache_files = {}
all_files = os.listdir(cache_dir_path)
for cache_file in all_files:
cache_file_path = os.path.join(cache_dir_path, cache_file)
if not os.path.isfile(cache_file_path):
continue
record = analyze_cache_file(cache_file, cache_file_path)
if not record:
continue
cache_files[record['name']] = record
return cache_files
#-------------------------------------------------------------------------------
[docs]def module_status(mod_data, pidf_data, proc_data, overall_status):
"""
Analyze status of given module.
"""
# Check the consistency of running processes and pid files.
consistent = True
if pidf_data:
for pid in pidf_data.keys():
if not proc_data or not pid in proc_data:
overall_status['messages']['error'].append((
"Process '{}' is dead, but pid file '{}' exists".format(
pid,
pidf_data[pid]['path']
),
'pidfile',
pidf_data[pid]['path']
))
consistent = False
if proc_data:
for pid in proc_data.keys():
if not pidf_data or not pid in pidf_data:
overall_status['messages']['error'].append((
"Process '{}':'{}' is running, but pid file is missing".format(
pid,
proc_data[pid]['name']
),
'process',
pid
))
consistent = False
# PID file exists and process is running.
if pidf_data and proc_data:
# Check the number of running processes
if mod_data.paralel:
if mod_data.count > len(proc_data.keys()):
return (
STATUS_RT_RUNNING_FEWER,
"There are fewer instances running than required ({}:{})".format(mod_data.count, len(proc_data.keys()))
)
if mod_data.count < len(proc_data.keys()):
return (
STATUS_RT_RUNNING_MORE,
"There are more instances running than required ({}:{})".format(mod_data.count, len(proc_data.keys()))
)
# Process is running or service is OK
if consistent:
return (
STATUS_RT_RUNNING_OK,
"Process is running or service is OK ({})".format(len(proc_data.keys()))
)
return (
STATUS_RT_RUNNING_INCONSISTENT,
"Process is running in inconsistent state"
)
# PID file(s) exist(s) and any process is not running
if pidf_data:
if mod_data.paralel:
return (
STATUS_RT_DEAD_PF_EXISTS,
"Processes are dead, but some pid files exist"
)
return (
STATUS_RT_DEAD_PF_EXISTS,
"Process is dead, but pid file exist"
)
# PID file(s) do(es) not exist, but some process is running
if proc_data:
if mod_data.paralel:
return (
STATUS_RT_RUNNING_PF_MISSING,
"Processes are running, but pid files are missing"
)
return (
STATUS_RT_RUNNING_PF_MISSING,
"Process is running, but pid file is missing"
)
# Process is not running
return (
STATUS_RT_NOT_RUNNING,
"Process is not running or service is stopped"
)
# TODO: Process is dead and /var/lock lock file exists
#return STATUS_RT_DEAD_LOCK_EXISTS
# TODO: Process or service status is unknown
#return STATUS_RT_UNKNOWN
[docs]def cronjob_status(cronj_data, cronf_data, overall_status):
"""
Analyze status of given cron module.
"""
if cronf_data and cronf_data['link']:
return (
STATUS_CJ_ENABLED,
"Cronjob is enabled"
)
return (
STATUS_CJ_DISABLED,
"Cronjob is disabled"
)
[docs]def system_status(modules, cronjobs, cfg_dir_path, cron_dir_path, log_dir_path, run_dir_path):
"""
Analyze status of all modules and cronjobs and detect additional problems.
"""
status = {}
status['dt'] = datetime.datetime.utcnow()
status['modules'] = {}
status['cronjobs'] = {}
status['messages'] = {'info': [], 'notice': [], 'warning': [], 'error': []}
status['processes'] = analyze_process_list_ps()
status['pid_files'] = analyze_pid_files(run_dir_path)
status['log_files'] = analyze_log_files(log_dir_path)
status['runlog_files'] = analyze_runlog_files(run_dir_path, limit = 1)
status['cron_files'] = analyze_cron_files(os.path.join(cfg_dir_path, 'cron'), cron_dir_path)
status['resultm'] = None
status['resultc'] = None
status['result'] = None
# A) Analyze status of configured modules.
for mname, mdata in modules.items():
status['modules'][mname] = module_status(
mdata,
status['pid_files'].get(mname, None),
status['processes'].get(mname, None),
status
)
# B) Analyze status of configured cronjob modules.
for mname, mdata in cronjobs.items():
status['cronjobs'][mname] = cronjob_status(
mdata,
status['cron_files'].get(mname, None),
status
)
# -----
_analyze_unconfigured_pid_files(status, modules)
_analyze_unconfigured_modules(status, modules)
_analyze_unconfigured_cronjobs(status, cronjobs)
# -----
# Determine overall status of real-time processing modules.
for mname, mstatus in status['modules'].items():
if not status['resultm']:
status['resultm'] = (mstatus[0], OVERALL_MODULE_STATUS_MESSAGES[mstatus[0]])
elif status['resultm'][0] != mstatus[0]:
status['resultm'] = (STATUS_RT_UNKNOWN, OVERALL_MODULE_STATUS_MESSAGES[STATUS_RT_UNKNOWN])
if not status['resultm']:
status['resultm'] = (STATUS_RT_NOT_RUNNING, OVERALL_MODULE_STATUS_MESSAGES[STATUS_RT_NOT_RUNNING])
# Determine overall status of cronjob modules.
for mname, mstatus in status['cronjobs'].items():
if not status['resultc']:
status['resultc'] = (mstatus[0], OVERALL_CRONJOB_STATUS_MESSAGES[mstatus[0]])
elif status['resultc'][0] != mstatus[0]:
status['resultc'] = (STATUS_CJ_UNKNOWN, OVERALL_CRONJOB_STATUS_MESSAGES[STATUS_CJ_UNKNOWN])
if not status['resultc']:
status['resultc'] = (STATUS_CJ_DISABLED, OVERALL_CRONJOB_STATUS_MESSAGES[STATUS_CJ_DISABLED])
# Determine overall status of the whole system.
if status['resultm'][0] != status['resultc'][0]:
status['result'] = (STATUS_UNKNOWN, OVERALL_STATUS_MESSAGES[STATUS_UNKNOWN])
else:
status['result'] = (status['resultm'][0], OVERALL_STATUS_MESSAGES[status['resultm'][0]])
return status
[docs]def analyze_versions():
"""
Analyze versions of various relevant dependencies (like PostgreSQL, ...).
"""
result = {}
out = subprocess.check_output(['psql','-V'])
result['postgresql'] = out.decode('utf-8').strip()
return result
#-------------------------------------------------------------------------------
def _analyze_unconfigured_pid_files(status, modules):
"""
Analyze list of all PID files and find any that do not belong to known modules.
"""
for mname in sorted(status['pid_files'].keys()):
if mname in modules:
continue
for pid in sorted(status['pid_files'][mname].keys()):
status['messages']['info'].append((
"Unknown pid file '{}' for PID '{}'".format(
status['pid_files'][mname][pid]['path'],
pid
),
'pidfile',
status['pid_files'][mname][pid]['path']
))
def _analyze_unconfigured_modules(status, modules):
"""
Analyze list of all processes files and find any that do not belong to known modules.
"""
for mname in sorted(status['processes'].keys()):
if mname in modules:
continue
for pid in sorted(status['processes'][mname].keys()):
status['messages']['info'].append((
"Unknown process '{}' with PID '{}'".format(
status['processes'][mname][pid]['exec'],
pid
),
'pidfile',
pid
))
def _analyze_unconfigured_cronjobs(status, cronjobs):
"""
Analyze list of all cron files and find any that do not belong to known cronjob
modules.
"""
for mname in sorted(status['cron_files'].keys()):
if mname in cronjobs:
continue
if status['cron_files'][mname]['link']:
status['messages']['info'].append((
"Unconfigured cron file '{}' for cron module '{}'".format(
status['cron_files'][mname]['link'],
status['cron_files'][mname]['path']
),
'cronfile',
status['cron_files'][mname]['path']
))