Source code for mentat.stats.test_rrd

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# This file is part of Mentat system (https://mentat.cesnet.cz/).
#
# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------


import unittest

import os
import time
import shutil
import random

import mentat.stats.rrd


RRD_STATS_DIR   = '/var/tmp/utest_rrdstats'
RRD_REPORTS_DIR = '/var/tmp'
TEST_DATA_SIZE  = 5000
TIME_START      = int(time.time())
TIME_START      = TIME_START - (TIME_START % mentat.stats.rrd.DFLT_STEP) - (TEST_DATA_SIZE * mentat.stats.rrd.DFLT_STEP)


[docs]class TestMentatStatsRrd(unittest.TestCase): # # Turn on more verbose output, which includes print-out of constructed # objects. This will really clutter your console, usable only for test # debugging. # verbose = False
[docs] def setUp(self): os.mkdir(RRD_STATS_DIR) self.stats = mentat.stats.rrd.RrdStats(RRD_STATS_DIR, RRD_REPORTS_DIR)
[docs] def tearDown(self): shutil.rmtree(RRD_STATS_DIR)
[docs] def test_01_internals(self): """ Perform the basic operativity tests of internal and helper methods. """ self.assertEqual(self.stats.clean('abcDEF123-_'), 'abcDEF123-_') self.assertEqual(self.stats.clean('abcDEF123-_<,>./?;:"[]{}()=+*!@#$%^&*'), 'abcDEF123-___________________________') self.assertEqual(self.stats._color_for_ds('typea', 'testa'), 'FF0000') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typea', 'testb'), 'FFFF00') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typea', 'testc'), '0000FF') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typea', 'testa'), 'FF0000') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typea', 'testb'), 'FFFF00') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typea', 'testc'), '0000FF') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typeb', 'testa'), 'FF0000') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typeb', 'testb'), 'FFFF00') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typeb', 'testc'), '0000FF') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typeb', 'testa'), 'FF0000') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typeb', 'testb'), 'FFFF00') # pylint: disable=locally-disabled,protected-access self.assertEqual(self.stats._color_for_ds('typeb', 'testc'), '0000FF') # pylint: disable=locally-disabled,protected-access
[docs] def test_02_prepare_db(self): """ Test creation of RRD database files. """ self.maxDiff = None tst = TIME_START - mentat.stats.rrd.DFLT_STEP # # Create RRD databases for three different datasets of two different types. # self.assertEqual(self.stats.prepare_db('typea.{}'.format(mentat.stats.rrd.DB_TOTALS_NAME), tst), ('/var/tmp/utest_rrdstats/typea._totals.rrd', True)) self.assertEqual(self.stats.prepare_db('typeb.{}'.format(mentat.stats.rrd.DB_TOTALS_NAME), tst), ('/var/tmp/utest_rrdstats/typeb._totals.rrd', True)) self.assertEqual(self.stats.prepare_db('typea.testa', tst), ('/var/tmp/utest_rrdstats/typea.testa.rrd', True)) self.assertEqual(self.stats.prepare_db('typea.testb', tst), ('/var/tmp/utest_rrdstats/typea.testb.rrd', True)) self.assertEqual(self.stats.prepare_db('typea.testc', tst), ('/var/tmp/utest_rrdstats/typea.testc.rrd', True)) self.assertEqual(self.stats.prepare_db('typeb.testa', tst), ('/var/tmp/utest_rrdstats/typeb.testa.rrd', True)) self.assertEqual(self.stats.prepare_db('typeb.testb', tst), ('/var/tmp/utest_rrdstats/typeb.testb.rrd', True)) self.assertEqual(self.stats.prepare_db('typeb.testc', tst), ('/var/tmp/utest_rrdstats/typeb.testc.rrd', True)) # # Create same RRD databases again, but the files already exist, and will not be created. # self.assertEqual(self.stats.prepare_db('typea.{}'.format(mentat.stats.rrd.DB_TOTALS_NAME), tst), ('/var/tmp/utest_rrdstats/typea._totals.rrd', False)) self.assertEqual(self.stats.prepare_db('typeb.{}'.format(mentat.stats.rrd.DB_TOTALS_NAME), tst), ('/var/tmp/utest_rrdstats/typeb._totals.rrd', False)) self.assertEqual(self.stats.prepare_db('typea.testa', tst), ('/var/tmp/utest_rrdstats/typea.testa.rrd', False)) self.assertEqual(self.stats.prepare_db('typea.testb', tst), ('/var/tmp/utest_rrdstats/typea.testb.rrd', False)) self.assertEqual(self.stats.prepare_db('typea.testc', tst), ('/var/tmp/utest_rrdstats/typea.testc.rrd', False)) self.assertEqual(self.stats.prepare_db('typeb.testa', tst), ('/var/tmp/utest_rrdstats/typeb.testa.rrd', False)) self.assertEqual(self.stats.prepare_db('typeb.testb', tst), ('/var/tmp/utest_rrdstats/typeb.testb.rrd', False)) self.assertEqual(self.stats.prepare_db('typeb.testc', tst), ('/var/tmp/utest_rrdstats/typeb.testc.rrd', False)) # # Check the existence of database files. # self.assertTrue(os.path.isfile(os.path.join(RRD_STATS_DIR, 'typea.{}.rrd'.format(mentat.stats.rrd.DB_TOTALS_NAME)))) self.assertTrue(os.path.isfile(os.path.join(RRD_STATS_DIR, 'typeb.{}.rrd'.format(mentat.stats.rrd.DB_TOTALS_NAME)))) self.assertTrue(os.path.isfile(os.path.join(RRD_STATS_DIR, 'typea.testa.rrd'))) self.assertTrue(os.path.isfile(os.path.join(RRD_STATS_DIR, 'typea.testb.rrd'))) self.assertTrue(os.path.isfile(os.path.join(RRD_STATS_DIR, 'typea.testc.rrd'))) self.assertTrue(os.path.isfile(os.path.join(RRD_STATS_DIR, 'typeb.testa.rrd'))) self.assertTrue(os.path.isfile(os.path.join(RRD_STATS_DIR, 'typeb.testb.rrd'))) self.assertTrue(os.path.isfile(os.path.join(RRD_STATS_DIR, 'typeb.testc.rrd')))
[docs] def test_03_find_dbs(self): """ Test the lookup of RRD database files. """ self.maxDiff = None self.test_02_prepare_db() self.assertEqual(self.stats.find_dbs(), { 'typea': [ ( 'typea._totals', 'typea', '_totals', '/var/tmp/utest_rrdstats/typea._totals.rrd', True ), ( 'typea.testa', 'typea', 'testa', '/var/tmp/utest_rrdstats/typea.testa.rrd', False ), ( 'typea.testb', 'typea', 'testb', '/var/tmp/utest_rrdstats/typea.testb.rrd', False ), ( 'typea.testc', 'typea', 'testc', '/var/tmp/utest_rrdstats/typea.testc.rrd', False ) ], 'typeb': [ ( 'typeb._totals', 'typeb', '_totals', '/var/tmp/utest_rrdstats/typeb._totals.rrd', True ), ( 'typeb.testa', 'typeb', 'testa', '/var/tmp/utest_rrdstats/typeb.testa.rrd', False ), ( 'typeb.testb', 'typeb', 'testb', '/var/tmp/utest_rrdstats/typeb.testb.rrd', False ), ( 'typeb.testc', 'typeb', 'testc', '/var/tmp/utest_rrdstats/typeb.testc.rrd', False ) ] }) self.assertEqual(self.stats.find_dbs('typea'), { 'typea': [ ( 'typea._totals', 'typea', '_totals', '/var/tmp/utest_rrdstats/typea._totals.rrd', True ), ( 'typea.testa', 'typea', 'testa', '/var/tmp/utest_rrdstats/typea.testa.rrd', False ), ( 'typea.testb', 'typea', 'testb', '/var/tmp/utest_rrdstats/typea.testb.rrd', False ), ( 'typea.testc', 'typea', 'testc', '/var/tmp/utest_rrdstats/typea.testc.rrd', False ) ] }) self.assertEqual(self.stats.find_dbs('typeb'), { 'typeb': [ ( 'typeb._totals', 'typeb', '_totals', '/var/tmp/utest_rrdstats/typeb._totals.rrd', True ), ( 'typeb.testa', 'typeb', 'testa', '/var/tmp/utest_rrdstats/typeb.testa.rrd', False ), ( 'typeb.testb', 'typeb', 'testb', '/var/tmp/utest_rrdstats/typeb.testb.rrd', False ), ( 'typeb.testc', 'typeb', 'testc', '/var/tmp/utest_rrdstats/typeb.testc.rrd', False ) ] })
[docs] def test_04_update(self): """ Test update of RRD database files. """ self.maxDiff = None self.test_02_prepare_db() rrd_dbs = self.stats.find_dbs() for idx in range(TEST_DATA_SIZE): tstamp = TIME_START + (idx * mentat.stats.rrd.DFLT_STEP) total = 0 for (rrddbt, rrddb_list) in rrd_dbs.items(): for rrddb in rrddb_list: value = random.randint(0, 1000) total += value if not rrddb[4]: self.assertEqual(self.stats.update(rrddb[0], value, tst=tstamp), (rrddb[3], False)) # Store summaries into '_totals' database. for rrddb in rrddb_list: if rrddb[4]: self.assertEqual(self.stats.update(rrddb[0], total, tst=tstamp), (rrddb[3], False))
[docs] def test_05_update_all(self): """ Test global update of all RRD database files. """ self.maxDiff = None self.test_02_prepare_db() for idx in range(TEST_DATA_SIZE): tstamp = TIME_START + (idx * mentat.stats.rrd.DFLT_STEP) self.assertEqual(self.stats.update_all(random.randint(0, 1000), tst=tstamp), [ ('/var/tmp/utest_rrdstats/typea._totals.rrd', False), ('/var/tmp/utest_rrdstats/typea.testa.rrd', False), ('/var/tmp/utest_rrdstats/typea.testb.rrd', False), ('/var/tmp/utest_rrdstats/typea.testc.rrd', False), ('/var/tmp/utest_rrdstats/typeb._totals.rrd', False), ('/var/tmp/utest_rrdstats/typeb.testa.rrd', False), ('/var/tmp/utest_rrdstats/typeb.testb.rrd', False), ('/var/tmp/utest_rrdstats/typeb.testc.rrd', False) ])
[docs] def test_06_export(self): """ Test exporting of RRD database files. """ self.maxDiff = None self.test_04_update() rrd_dbs = self.stats.find_dbs() for (rrddbt, rrddb_list) in rrd_dbs.items(): for rrddb in rrddb_list: (rrddbf, flag_new, result) = self.stats.export(rrddb[0]) self.assertEqual(rrddbf, rrddb[3]) self.assertFalse(flag_new) self.assertTrue(result)
[docs] def test_07_lookup(self): """ Test lookup of all RRD charts, spark charts and JSON export files. """ self.test_04_update() result = self.stats.lookup()
[docs] def test_08_generate(self): """ Test generating all RRD charts, spark charts and JSON export files. """ self.test_04_update() time_end = (TIME_START + (mentat.stats.rrd.DFLT_STEP * TEST_DATA_SIZE)) result = self.stats.generate(time_end) for res in result: self.assertTrue(os.path.isfile(res)) self.assertEqual(result, [ '/var/tmp/typea.l6hours.meta.json', '/var/tmp/typea.l6hours.png', '/var/tmp/typea.l6hours.spark.png', '/var/tmp/typea.l6hours.xport.json', '/var/tmp/typea.l6hours-t.meta.json', '/var/tmp/typea.l6hours-t.png', '/var/tmp/typea.l6hours-t.spark.png', '/var/tmp/typea.l6hours-t.xport.json', '/var/tmp/typea.l24hours.meta.json', '/var/tmp/typea.l24hours.png', '/var/tmp/typea.l24hours.spark.png', '/var/tmp/typea.l24hours.xport.json', '/var/tmp/typea.l24hours-t.meta.json', '/var/tmp/typea.l24hours-t.png', '/var/tmp/typea.l24hours-t.spark.png', '/var/tmp/typea.l24hours-t.xport.json', '/var/tmp/typea.l72hours.meta.json', '/var/tmp/typea.l72hours.png', '/var/tmp/typea.l72hours.spark.png', '/var/tmp/typea.l72hours.xport.json', '/var/tmp/typea.l72hours-t.meta.json', '/var/tmp/typea.l72hours-t.png', '/var/tmp/typea.l72hours-t.spark.png', '/var/tmp/typea.l72hours-t.xport.json', '/var/tmp/typea.lweek.meta.json', '/var/tmp/typea.lweek.png', '/var/tmp/typea.lweek.spark.png', '/var/tmp/typea.lweek.xport.json', '/var/tmp/typea.lweek-t.meta.json', '/var/tmp/typea.lweek-t.png', '/var/tmp/typea.lweek-t.spark.png', '/var/tmp/typea.lweek-t.xport.json', '/var/tmp/typea.l2weeks.meta.json', '/var/tmp/typea.l2weeks.png', '/var/tmp/typea.l2weeks.spark.png', '/var/tmp/typea.l2weeks.xport.json', '/var/tmp/typea.l2weeks-t.meta.json', '/var/tmp/typea.l2weeks-t.png', '/var/tmp/typea.l2weeks-t.spark.png', '/var/tmp/typea.l2weeks-t.xport.json', '/var/tmp/typea.l4weeks.meta.json', '/var/tmp/typea.l4weeks.png', '/var/tmp/typea.l4weeks.spark.png', '/var/tmp/typea.l4weeks.xport.json', '/var/tmp/typea.l4weeks-t.meta.json', '/var/tmp/typea.l4weeks-t.png', '/var/tmp/typea.l4weeks-t.spark.png', '/var/tmp/typea.l4weeks-t.xport.json', '/var/tmp/typea.l3months.meta.json', '/var/tmp/typea.l3months.png', '/var/tmp/typea.l3months.spark.png', '/var/tmp/typea.l3months.xport.json', '/var/tmp/typea.l3months-t.meta.json', '/var/tmp/typea.l3months-t.png', '/var/tmp/typea.l3months-t.spark.png', '/var/tmp/typea.l3months-t.xport.json', '/var/tmp/typea.l6months.meta.json', '/var/tmp/typea.l6months.png', '/var/tmp/typea.l6months.spark.png', '/var/tmp/typea.l6months.xport.json', '/var/tmp/typea.l6months-t.meta.json', '/var/tmp/typea.l6months-t.png', '/var/tmp/typea.l6months-t.spark.png', '/var/tmp/typea.l6months-t.xport.json', '/var/tmp/typea.lyear.meta.json', '/var/tmp/typea.lyear.png', '/var/tmp/typea.lyear.spark.png', '/var/tmp/typea.lyear.xport.json', '/var/tmp/typea.lyear-t.meta.json', '/var/tmp/typea.lyear-t.png', '/var/tmp/typea.lyear-t.spark.png', '/var/tmp/typea.lyear-t.xport.json', '/var/tmp/typea.l2years.meta.json', '/var/tmp/typea.l2years.png', '/var/tmp/typea.l2years.spark.png', '/var/tmp/typea.l2years.xport.json', '/var/tmp/typea.l2years-t.meta.json', '/var/tmp/typea.l2years-t.png', '/var/tmp/typea.l2years-t.spark.png', '/var/tmp/typea.l2years-t.xport.json', '/var/tmp/typeb.l6hours.meta.json', '/var/tmp/typeb.l6hours.png', '/var/tmp/typeb.l6hours.spark.png', '/var/tmp/typeb.l6hours.xport.json', '/var/tmp/typeb.l6hours-t.meta.json', '/var/tmp/typeb.l6hours-t.png', '/var/tmp/typeb.l6hours-t.spark.png', '/var/tmp/typeb.l6hours-t.xport.json', '/var/tmp/typeb.l24hours.meta.json', '/var/tmp/typeb.l24hours.png', '/var/tmp/typeb.l24hours.spark.png', '/var/tmp/typeb.l24hours.xport.json', '/var/tmp/typeb.l24hours-t.meta.json', '/var/tmp/typeb.l24hours-t.png', '/var/tmp/typeb.l24hours-t.spark.png', '/var/tmp/typeb.l24hours-t.xport.json', '/var/tmp/typeb.l72hours.meta.json', '/var/tmp/typeb.l72hours.png', '/var/tmp/typeb.l72hours.spark.png', '/var/tmp/typeb.l72hours.xport.json', '/var/tmp/typeb.l72hours-t.meta.json', '/var/tmp/typeb.l72hours-t.png', '/var/tmp/typeb.l72hours-t.spark.png', '/var/tmp/typeb.l72hours-t.xport.json', '/var/tmp/typeb.lweek.meta.json', '/var/tmp/typeb.lweek.png', '/var/tmp/typeb.lweek.spark.png', '/var/tmp/typeb.lweek.xport.json', '/var/tmp/typeb.lweek-t.meta.json', '/var/tmp/typeb.lweek-t.png', '/var/tmp/typeb.lweek-t.spark.png', '/var/tmp/typeb.lweek-t.xport.json', '/var/tmp/typeb.l2weeks.meta.json', '/var/tmp/typeb.l2weeks.png', '/var/tmp/typeb.l2weeks.spark.png', '/var/tmp/typeb.l2weeks.xport.json', '/var/tmp/typeb.l2weeks-t.meta.json', '/var/tmp/typeb.l2weeks-t.png', '/var/tmp/typeb.l2weeks-t.spark.png', '/var/tmp/typeb.l2weeks-t.xport.json', '/var/tmp/typeb.l4weeks.meta.json', '/var/tmp/typeb.l4weeks.png', '/var/tmp/typeb.l4weeks.spark.png', '/var/tmp/typeb.l4weeks.xport.json', '/var/tmp/typeb.l4weeks-t.meta.json', '/var/tmp/typeb.l4weeks-t.png', '/var/tmp/typeb.l4weeks-t.spark.png', '/var/tmp/typeb.l4weeks-t.xport.json', '/var/tmp/typeb.l3months.meta.json', '/var/tmp/typeb.l3months.png', '/var/tmp/typeb.l3months.spark.png', '/var/tmp/typeb.l3months.xport.json', '/var/tmp/typeb.l3months-t.meta.json', '/var/tmp/typeb.l3months-t.png', '/var/tmp/typeb.l3months-t.spark.png', '/var/tmp/typeb.l3months-t.xport.json', '/var/tmp/typeb.l6months.meta.json', '/var/tmp/typeb.l6months.png', '/var/tmp/typeb.l6months.spark.png', '/var/tmp/typeb.l6months.xport.json', '/var/tmp/typeb.l6months-t.meta.json', '/var/tmp/typeb.l6months-t.png', '/var/tmp/typeb.l6months-t.spark.png', '/var/tmp/typeb.l6months-t.xport.json', '/var/tmp/typeb.lyear.meta.json', '/var/tmp/typeb.lyear.png', '/var/tmp/typeb.lyear.spark.png', '/var/tmp/typeb.lyear.xport.json', '/var/tmp/typeb.lyear-t.meta.json', '/var/tmp/typeb.lyear-t.png', '/var/tmp/typeb.lyear-t.spark.png', '/var/tmp/typeb.lyear-t.xport.json', '/var/tmp/typeb.l2years.meta.json', '/var/tmp/typeb.l2years.png', '/var/tmp/typeb.l2years.spark.png', '/var/tmp/typeb.l2years.xport.json', '/var/tmp/typeb.l2years-t.meta.json', '/var/tmp/typeb.l2years-t.png', '/var/tmp/typeb.l2years-t.spark.png', '/var/tmp/typeb.l2years-t.xport.json' ])
if __name__ == "__main__": unittest.main()