Source code for mentat.reports.utils

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# This file is part of Mentat system (https://mentat.cesnet.cz/).
#
# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------


"""
Library containing reporting utilities.
"""


__author__  = "Jan Mach <jan.mach@cesnet.cz>"
__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"


import datetime
import pytz
import pprint

from pynspect.jpath import jpath_value, jpath_values

import mentat.const
import mentat.datatype.internal


[docs]class ReportingSettings: # pylint: disable=locally-disabled,too-many-instance-attributes """ Class for custom manipulations with abuse group reporting settings. """ def __init__(self, abuse_group, **kwargs): self.group_name = abuse_group.name self.filters = abuse_group.filters self.networks = abuse_group.networks self.emails = self._init_emails(abuse_group.name, abuse_group.settings_rep.emails) self.mode = self._init_mode(abuse_group.settings_rep.mode, **kwargs) self.attachments = self._init_attachments(abuse_group.settings_rep.attachments, **kwargs) self.template = self._init_template(abuse_group.settings_rep.template, **kwargs) self.locale = self._init_locale(abuse_group.settings_rep.locale, **kwargs) self.timezone = self._init_timezone(abuse_group.settings_rep.timezone, **kwargs) self.timing = self._init_timing(abuse_group.settings_rep.timing, **kwargs) self.timing_cfg = self._init_timing_cfg(abuse_group.settings_rep) self.mute = self._init_mute(abuse_group.settings_rep.mute, **kwargs) self.redirect = self._init_redirect(abuse_group.settings_rep.redirect, **kwargs) self.compress = self._init_compress(abuse_group.settings_rep.compress, **kwargs) self.max_size = self._init_maxattachsize(abuse_group.settings_rep.max_attachment_size, **kwargs) def __repr__(self): return 'ReportingSettings(group_name={};filters={};networks={};emails={};mode={};attachments={};template={};locale={};timezone={};timing={};timing_cfg={};mute={};redirect={};compress={};max_size={})'.format( self.group_name, pprint.pformat(self.filters, compact = True), pprint.pformat(self.networks, compact = True), pprint.pformat(self.emails, compact = True), pprint.pformat(self.mode, compact = True), pprint.pformat(self.attachments, compact = True), pprint.pformat(self.template, compact = True), pprint.pformat(self.locale, compact = True), pprint.pformat(self.timezone, compact = True), pprint.pformat(self.timing, compact = True), pprint.pformat(self.timing_cfg, compact = True, width = 10000), pprint.pformat(self.mute, compact = True), pprint.pformat(self.redirect, compact = True), pprint.pformat(self.compress, compact = True), pprint.pformat(self.max_size, compact = True) ) @staticmethod def _init_emails(group_value, settings_emails): if settings_emails: return settings_emails return [group_value] @staticmethod def _init_mode(group_value, **kwargs): if 'force_mode' in kwargs and kwargs['force_mode'] is not None: if kwargs['force_mode'] not in mentat.const.REPORTING_MODES: raise ValueError("Invalid value '{:s}' for reporting mode.".format(kwargs['force_mode'])) return str(kwargs['force_mode']) if group_value is not None: return group_value return mentat.const.DFLT_REPORTING_MODE @staticmethod def _init_attachments(group_value, **kwargs): if 'force_attachments' in kwargs and kwargs['force_attachments'] is not None: if kwargs['force_attachments'] not in mentat.const.REPORTING_ATTACHS: raise ValueError("Invalid value '{}' for reporting attachments.".format(kwargs['force_attachments'])) return str(kwargs['force_attachments']) if group_value is not None: return group_value return mentat.const.DFLT_REPORTING_ATTACHMENTS @staticmethod def _init_template(group_value, **kwargs): if 'force_template' in kwargs and kwargs['force_template'] is not None: return str(kwargs['force_template']) if group_value is not None: return group_value return mentat.const.DFLT_REPORTING_TEMPLATE @staticmethod def _init_locale(group_value, **kwargs): if 'force_locale' in kwargs and kwargs['force_locale'] is not None: return str(kwargs['force_locale']) if group_value is not None: return group_value return mentat.const.DFLT_REPORTING_LOCALE @staticmethod def _init_timezone(group_value, **kwargs): if 'force_timezone' in kwargs and kwargs['force_timezone'] is not None: if kwargs['force_timezone'] not in pytz.common_timezones: raise ValueError("Invalid value '{}' for reporting timezone. Please use one of the values defined in pytz.common_timezones package.".format(kwargs['force_timezone'])) return str(kwargs['force_timezone']) if group_value is not None: return group_value return mentat.const.DFLT_REPORTING_TIMEZONE @staticmethod def _init_mute(group_value, **kwargs): if 'force_mute' in kwargs and kwargs['force_mute'] is not None: return bool(kwargs['force_mute']) if group_value is not None: return group_value return mentat.const.DFLT_REPORTING_MUTE @staticmethod def _init_redirect(group_value, **kwargs): if 'force_redirect' in kwargs and kwargs['force_redirect'] is not None: return bool(kwargs['force_redirect']) if group_value is not None: return group_value return mentat.const.DFLT_REPORTING_REDIRECT @staticmethod def _init_compress(group_value, **kwargs): if 'force_compress' in kwargs and kwargs['force_compress'] is not None: return bool(kwargs['force_compress']) if group_value is not None: return group_value return mentat.const.DFLT_REPORTING_COMPRESS @staticmethod def _init_timing(group_value, **kwargs): if 'force_timing' in kwargs and kwargs['force_timing'] is not None: if kwargs['force_timing'] not in mentat.const.REPORTING_TIMINGS: raise ValueError("Invalid value '{}' for reporting timing.".format(kwargs['force_timing'])) return str(kwargs['force_timing']) if group_value is not None: return group_value return mentat.const.DFLT_REPORTING_TIMING @staticmethod def _init_maxattachsize(group_value, **kwargs): if 'force_maxattachsize' in kwargs and kwargs['force_maxattachsize'] is not None: return int(kwargs['force_maxattachsize']) if group_value is not None: return group_value return mentat.const.DFLT_REPORTING_MAXATTACHSIZE @staticmethod def _init_timing_cfg(settings_rep): if settings_rep.timing == mentat.const.REPORTING_TIMING_DEFAULT: return { mentat.const.EVENT_SEVERITY_LOW: { 'per': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_LOW_PER] ), 'thr': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_LOW_THR] ), 'rel': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_LOW_REL] ) }, mentat.const.EVENT_SEVERITY_MEDIUM: { 'per': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_MEDIUM_PER] ), 'thr': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_MEDIUM_THR] ), 'rel': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_MEDIUM_REL] ) }, mentat.const.EVENT_SEVERITY_HIGH: { 'per': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_HIGH_PER] ), 'thr': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_HIGH_THR] ), 'rel': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_HIGH_REL] ) }, mentat.const.EVENT_SEVERITY_CRITICAL: { 'per': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_CRITICAL_PER] ), 'thr': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_CRITICAL_THR] ), 'rel': datetime.timedelta( seconds = mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_CRITICAL_REL] ) } } return { mentat.const.EVENT_SEVERITY_LOW: { 'per': datetime.timedelta( seconds = settings_rep.timing_per_lo or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_LOW_PER] ), 'thr': datetime.timedelta( seconds = settings_rep.timing_thr_lo or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_LOW_THR] ), 'rel': datetime.timedelta( seconds = settings_rep.timing_rel_lo or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_LOW_REL] ) }, mentat.const.EVENT_SEVERITY_MEDIUM: { 'per': datetime.timedelta( seconds = settings_rep.timing_per_md or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_MEDIUM_PER] ), 'thr': datetime.timedelta( seconds = settings_rep.timing_thr_md or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_MEDIUM_THR] ), 'rel': datetime.timedelta( seconds = settings_rep.timing_rel_md or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_MEDIUM_REL] ) }, mentat.const.EVENT_SEVERITY_HIGH: { 'per': datetime.timedelta( seconds = settings_rep.timing_per_hi or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_HIGH_PER] ), 'thr': datetime.timedelta( seconds = settings_rep.timing_thr_hi or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_HIGH_THR] ), 'rel': datetime.timedelta( seconds = settings_rep.timing_rel_hi or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_HIGH_REL] ) }, mentat.const.EVENT_SEVERITY_CRITICAL: { 'per': datetime.timedelta( seconds = settings_rep.timing_per_cr or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_CRITICAL_PER] ), 'thr': datetime.timedelta( seconds = settings_rep.timing_thr_cr or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_CRITICAL_THR] ), 'rel': datetime.timedelta( seconds = settings_rep.timing_rel_cr or mentat.const.REPORTING_INTERVALS[mentat.const.REPORTING_TIMING_DEFAULT_CRITICAL_REL] ) } }
[docs] def setup_filters(self, flt_parser, flt_compiler): """ Setup and return list of filters in format appropriate for direct filtering by :py:func:`mentat.reports.event.EventReporter.filter_events` function. :param pynspect.gparser.PynspectFilterParser flt_parser: Parser object. :param pynspect.compilers.IDEAFilterCompiler flt_compiler: Compiler object. :return: List of processed and compiled filters. :rtype: list """ result = [] for filter_obj in self.filters: if not filter_obj.enabled: continue dt_now = datetime.datetime.utcnow() if filter_obj.valid_from and dt_now < filter_obj.valid_from: continue if filter_obj.valid_to and dt_now > filter_obj.valid_to: continue flt = flt_parser.parse(filter_obj.filter) flt = flt_compiler.compile(flt) result.append((filter_obj, flt)) return result
[docs] def setup_networks(self): """ Setup and return list of network in format appropriate for populating the :py:class:`mentat.services.whois.WhoisModule`. :return: List of processed networks. :rtype: list """ result = [] for net in self.networks: result.append(mentat.datatype.internal.t_network_record({ 'network': net.network, 'resolved_abuses': [self.group_name] })) return result
[docs]class ThresholdingCache: """ Base class for implementing event thresholding caches for periodical event reporting. """
[docs] def event_is_thresholded(self, event, ttl): """ Check, that given event is thresholded within given TTL. :param mentat.idea.internal.Idea event: IDEA event to check. :param datetime.datetime ttl: TTL for the thresholding record. :return: ``True`` in case the event is thresholded, ``False`` otherwise. :rtype: bool """ cachekeys = self._generate_cache_keys(event) result = True for chk in cachekeys: res = self.check(chk, ttl) if not res: result = False return result
[docs] def set_threshold(self, event, source, thresholdtime, relapsetime, ttl): """ Threshold given event with given TTL. :param mentat.idea.internal.Idea event: IDEA event to threshold. :param str source: Source address because of which to threshold the event. :param datetime.datetime thresholdtime: Threshold window start time. :param datetime.datetime relapsetime: Relapse window start time. :param datetime.datetime ttl: Record TTL. """ if source: source = [source] cachekeys = self._generate_cache_keys(event, source) for chk in cachekeys: self.set(chk, thresholdtime, relapsetime, ttl)
[docs] def threshold_event(self, event, group_name, severity, createtime): """ Threshold given event with given TTL. :param mentat.idea.internal.Idea event: IDEA event to threshold. :param str group_name: Name of the group for which to threshold. :param str severity: Event severity. :param datetime.datetime createtime: Thresholding timestamp. """ cachekeys = self._generate_cache_keys(event) for chk in cachekeys: self.save(event.get_id(), chk, group_name, severity, createtime)
#---------------------------------------------------------------------------
[docs] def check(self, key, ttl): """ Check event thresholding cache for given key and TTL. This method always returns ``False``. :param str key: Thresholding cache key. :param datetime.datetime ttl: Cache record TTL. :return: ``True`` if given key was found with valid TTL,``False`` othrewise. :rtype: bool """ raise NotImplementedError()
[docs] def set(self, key, thresholdtime, relapsetime, ttl): """ Set thresholding cache record with given key and TTL. :param str key: Thresholding cache key. :param datetime.datetime thresholdtime: Threshold window start time. :param datetime.datetime relapsetime: Relapse window start time. :param datetime.datetime ttl: Record TTL. """ raise NotImplementedError()
[docs] def save(self, event_id, key_id, group_name, severity, createtime): """ Save event into registry of thresholded events. :param str event_id: Event ID. :param str key_id: Thresholding cache key. :param datetime.datetime createtime: Time of the thresholding. """ raise NotImplementedError()
[docs] def relapses(self, group_name, severity, ttl): """ Search for list of relapsed events for given group and severity. :param str group_name: Name of the abuse group. :param str severity: Event severity. :param datetime.datetime ttl: Record TTL time. :return: List of relapsed events as :py:class:`mentat.idea.internal.Idea` objects. :rtype: list """ raise NotImplementedError()
[docs] def cleanup(self, ttl): """ Cleanup records from thresholding cache with TTL older than given value. :param datetime.datetime ttl: Record TTL cleanup threshold. """ raise NotImplementedError()
#--------------------------------------------------------------------------- def _generate_cache_keys(self, event, sources = None): """ Generate list of thresholding/relapse cache keys for given event. :param mentat.idea.internal.Idea event: Event to process. :return: List of cache keys as strings. :rtype: list """ keys = [] event_class = jpath_value(event, '_CESNET.EventClass') if not event_class: event_class = '/'.join(jpath_values(event, 'Category')) if not sources: sources_ip4 = jpath_values(event, 'Source.IP4') sources_ip6 = jpath_values(event, 'Source.IP6') sources = sources_ip4 + sources_ip6 for src in sources: keys.append('+++'.join((event_class, str(src)))) return keys
[docs]class NoThresholdingCache(ThresholdingCache): """ Implementation of the :py:class:`mentat.reports.utils.ThresholdingCache` that does no thresholding at all. It can be used to disable the thresholding feature during reporting, for example for generating some kind of ad-hoc reports. """
[docs] def check(self, key, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.check` method. """ return False
[docs] def set(self, key, thresholdtime, relapsetime, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.set` method. """ return
[docs] def save(self, event_id, key_id, group_name, severity, createtime): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.save` method. """ return
[docs] def relapses(self, group_name, severity, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.relapses` method. """ return []
[docs] def cleanup(self, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.cleanup` method. """ return {'thresholds': 0, 'events': 0}
[docs]class SimpleMemoryThresholdingCache(ThresholdingCache): """ Implementation of the :py:class:`mentat.reports.utils.ThresholdingCache` that performs thresholding within the memory structures. """ def __init__(self): self.memcache = {}
[docs] def check(self, key, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.check` method. """ return bool(key in self.memcache)
[docs] def set(self, key, thresholdtime, relapsetime, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.set` method. """ self.memcache[key] = True
[docs] def save(self, event_id, key_id, group_name, severity, createtime): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.save` method. """ return
[docs] def relapses(self, group_name, severity, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.relapses` method. """ return []
[docs] def cleanup(self, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.cleanup` method. """ return {'thresholds': 0, 'events': 0}
[docs]class SingleSourceThresholdingCache(SimpleMemoryThresholdingCache): """ Implementation of the :py:class:`mentat.reports.utils.ThresholdingCache` that performs thresholding within the memory structures. """ def __init__(self, source): super().__init__() self.source = source def _generate_cache_keys(self, event, sources = None): """ Generate list of thresholding/relapse cache keys for given event. :param mentat.idea.internal.Idea event: Event to process. :return: List of cache keys as strings. :rtype: list """ return super()._generate_cache_keys(event, [self.source])
[docs]class StorageThresholdingCache(ThresholdingCache): """ Implementation of the :py:class:`mentat.reports.utils.ThresholdingCache` that is using :py:class:`mentat.services.eventstorage` service for storing thresholding records. """ def __init__(self, logger, eventservice): self.logger = logger self.eventservice = eventservice self.memcache = {}
[docs] def check(self, key, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.check` method. """ if not key in self.memcache: result = self.eventservice.threshold_check(key, ttl) self.memcache[key] = bool(result) return self.memcache[key]
[docs] def set(self, key, thresholdtime, relapsetime, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.set` method. """ if not key in self.memcache or not self.memcache[key]: try: self.eventservice.threshold_set(key, thresholdtime, relapsetime, ttl) self.logger.info( "Updated thresholding cache with record - TTL=%s|RLP=%s|THR=%s|KEY=%s", ttl.isoformat(), relapsetime.isoformat(), thresholdtime.isoformat(), key ) except mentat.services.eventstorage.StorageIntegrityError: self.logger.info( "Prolonged thresholding cache record - TTL=%s|RLP=%s|THR=%s|KEY=%s", ttl.isoformat(), relapsetime.isoformat(), thresholdtime.isoformat(), key ) self.memcache[key] = True
[docs] def save(self, event_id, key_id, group_name, severity, createtime): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.save` method. """ try: self.eventservice.threshold_save(event_id, key_id, group_name, severity, createtime) self.logger.info( "Recorded thresholded event with record - CT=%s|KEY=%s|EID=%s|GRP=%s|SEV=%s", createtime.isoformat(), key_id, event_id, group_name, severity ) except mentat.services.eventstorage.StorageIntegrityError: self.logger.info( "Event is already thresholded with record - CT=%s|KEY=%s|EID=%s|GRP=%s|SEV=%s", createtime.isoformat(), key_id, event_id, group_name, severity )
[docs] def relapses(self, group_name, severity, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.relapses` method. """ return self.eventservice.search_relapsed_events(group_name, severity, ttl)
[docs] def cleanup(self, ttl): """ *Interface implementation:* Implementation of :py:func:`mentat.reports.utils.ThresholdingCache.cleanup` method. """ self.memcache = {} count_tc = self.eventservice.thresholds_clean(ttl) self.logger.info( "Cleaned %d records from thresholding cache older than %s.", count_tc, ttl.isoformat() ) count_te = self.eventservice.thresholded_events_clean() self.logger.info( "Cleaned %d records from registry of thresholded events.", count_te, ) return { 'thresholds': count_tc, 'events': count_te }