#!/usr/bin/python3 import amulet import unittest import time import yaml from charmhelpers.contrib.amulet.utils import ( AmuletUtils, ) autils = AmuletUtils() PAUSE_TIME = 30 class TestDeployment(unittest.TestCase): @classmethod def setUpClass(cls): cls.deployment = amulet.Deployment(series='trusty') cls.deployment.add('mysql') cls.deployment.add('nrpe') cls.deployment.add('nagios') cls.deployment.configure('mysql', {'dataset-size': '10%'}) cls.deployment.relate('nrpe:monitors', 'nagios:monitors') cls.deployment.relate('nrpe:local-monitors', 'mysql:local-monitors') cls.deployment.expose('nagios') try: cls.deployment.setup(timeout=900) cls.deployment.sentry.wait() except amulet.helpers.TimeoutError: msg = "Environment wasn't stood up in time" amulet.raise_status(amulet.SKIP, msg=msg) except Exception: raise def check_nrpe_setting(self, filename, expected_settings, juju_kv, filedelim=None): self.nrpe_sentry = self.deployment.sentry['nrpe'][0] if juju_kv: self.deployment.configure('nrpe', juju_kv) time.sleep(PAUSE_TIME) nrpe_contents = self.nrpe_sentry.file_contents(filename) for line in nrpe_contents.split('\n'): if not line: continue line_key = line.split(filedelim)[0] if line_key in expected_settings.keys(): line_value = ' '.join(line.split(filedelim)[1:]) if line_value != expected_settings[line_key]: msg = ('Setting %s in %s did not contain expected value ' '(%s != %s)') % (line_key, filename, line_value, expected_settings[line_key]) amulet.raise_status(amulet.FAIL, msg=msg) def test_monitors_relation_sub_monitors(self): """ Check default monitor definitions are passed to Nagios """ self.deployment.configure('nrpe', {'monitors': ''}) time.sleep(PAUSE_TIME) self.nrpe_sentry = self.deployment.sentry['nrpe'][0] relation_data = self.nrpe_sentry.relation( 'monitors', 'nagios:monitors', ) monitors = yaml.safe_load(relation_data['monitors']) checks = [ 'check_mem_sub', 'check_disk_root_sub', 'check_swap_sub', 'check_load_sub', 'check_users_sub', 'check_zombie_procs_sub', 'check_total_procs_sub', 'check_conntrack_sub', 'check_swap_activity_sub', ] for check in checks: if check not in monitors['monitors']['remote']['nrpe'].keys(): amulet.raise_status( amulet.FAIL, msg='{} not found in monitors relation'.format(check) ) def test_monitors_relation_principal_monitors(self): """ Check monitor definitions from principal are passed to Nagios """ self.nrpe_sentry = self.deployment.sentry['nrpe'][0] relation_data = self.nrpe_sentry.relation( 'monitors', 'nagios:monitors', ) monitors = yaml.safe_load(relation_data['monitors']) if 'mysql' not in monitors['monitors']['remote'].keys(): amulet.raise_status( amulet.FAIL, msg='mysql remote monitor not found in monitors relation', ) nrpe_checks = monitors['monitors']['remote']['nrpe'].keys() if 'check_proc_mysqld_principal' not in nrpe_checks: amulet.raise_status( amulet.FAIL, msg='mysql process monitor not found in monitors relation', ) def test_monitors_relation_user_monitors(self): """ Check user configured monitor definitions are passed to Nagios """ user_monitors = { 'version': '0.3', 'monitors': { 'local': { 'procrunning': { 'rsync': { 'max': 1, 'executable': 'rsync', 'name': 'RSYNc Running', 'min': 1 }, 'jujud': { 'max': 1, 'executable': 'jujud', 'name': 'Juju Running', 'min': 1 } } }, 'remote': { 'tcp': { 'ssh': { 'warning': 2, 'critical': 10, 'name': 'SSH Running', 'timeout': 12, 'port': 22, 'string': 'SSH.*', 'expect': None } } } } } self.deployment.configure( 'nrpe', {'monitors': yaml.dump(user_monitors)} ) time.sleep(PAUSE_TIME) self.nrpe_sentry = self.deployment.sentry['nrpe'][0] relation_data = self.nrpe_sentry.relation( 'monitors', 'nagios:monitors', ) monitors = yaml.safe_load(relation_data['monitors']) checks = ['check_proc_jujud_user', 'check_proc_jujud_user'] for check in checks: if check not in monitors['monitors']['remote']['nrpe'].keys(): amulet.raise_status( amulet.FAIL, msg='{} not found in monitors relation'.format(check), ) if 'ssh' not in monitors['monitors']['remote']['tcp'].keys(): amulet.raise_status( amulet.FAIL, msg='{} not found in monitors relation'.format(check), ) def test_services(self): """ Test basic services are running """ self.nagios_sentry = self.deployment.sentry['nagios'][0] self.nrpe_sentry = self.deployment.sentry['nrpe'][0] commands = { self.nrpe_sentry: ['service nagios-nrpe-server status'], self.nagios_sentry: ['service nagios3 status'], } ret = autils.validate_services(commands) if ret: amulet.raise_status(amulet.FAIL, msg=ret) def test_config_nagios_master(self): unit = self.deployment.sentry['nagios'][0] nagios_relation = unit.relation('monitors', 'nrpe:monitors') ipaddr = nagios_relation['private-address'] test_config = { 'filename': '/etc/nagios/nrpe.cfg', 'expected_settings': { 'allowed_hosts': '127.0.0.1,10.0.0.10,' + ipaddr }, 'juju_kv': {'nagios_master': '10.0.0.10'}, 'filedelim': '=', } self.check_nrpe_setting(**test_config) def test_config_rsync_fragment(self): test_config = { 'filename': '/etc/rsync-juju.d/010-nrpe-external-master.conf', 'expected_settings': {'hosts allow': '10.0.0.10'}, 'juju_kv': { 'nagios_master': '10.0.0.10', 'export_nagios_definitions': True, }, 'filedelim': '=', } self.check_nrpe_setting(**test_config) def test_config_rsync(self): test_config = { 'filename': '/etc/rsyncd.conf', 'expected_settings': {'&include': '/etc/rsync-juju.d'}, 'juju_kv': {'export_nagios_definitions': True}, } self.check_nrpe_setting(**test_config) def test_config_server_port(self): test_config = { 'filename': '/etc/nagios/nrpe.cfg', 'expected_settings': {'server_port': '5888'}, 'juju_kv': {'server_port': '5888'}, 'filedelim': '=', } self.check_nrpe_setting(**test_config) self.deployment.configure('nrpe', {'server_port': '5888'}) def test_config_debug(self): test_config = { 'filename': '/etc/nagios/nrpe.cfg', 'expected_settings': {'debug': '1'}, 'juju_kv': {'debug': 'True'}, 'filedelim': '=', } self.check_nrpe_setting(**test_config) self.deployment.configure('nrpe', {'debug': 'True'}) def test_config_dont_blame_nrpe(self): test_config = { 'filename': '/etc/nagios/nrpe.cfg', 'expected_settings': {'dont_blame_nrpe': '1'}, 'juju_kv': {'dont_blame_nrpe': 'True'}, 'filedelim': '=', } self.check_nrpe_setting(**test_config) self.deployment.configure('nrpe', {'dont_blame_nrpe': 'True'}) def test_nagios_host_context(self): hostname = 'bob-mysql-0' test_config = { 'filename': '/var/lib/nagios/export/host__%s.cfg' % (hostname), 'expected_settings': {'host_name': hostname, 'use': 'masterhostgroup', 'hostgroups': 'machines, myhostgroup1'}, 'juju_kv': { 'nagios_host_context': 'bob', 'hostcheck_inherit': 'masterhostgroup', 'nagios_hostname_type': 'unit', 'hostgroups': 'myhostgroup1', 'export_nagios_definitions': True, }, } self.check_nrpe_setting(**test_config) def test_nagios_hostname_type(self): sentry = self.deployment.sentry['nrpe'][0] hostname = sentry.run('hostname')[0] test_config = { 'filename': '/var/lib/nagios/export/host__%s.cfg' % (hostname), 'expected_settings': {'host_name': hostname, 'use': 'masterhostgroup', 'hostgroups': 'machines, myhostgroup1'}, 'juju_kv': { 'nagios_host_context': 'bob', 'hostcheck_inherit': 'masterhostgroup', 'hostgroups': 'myhostgroup1', 'nagios_hostname_type': 'host', 'export_nagios_definitions': True }, } self.check_nrpe_setting(**test_config) def test_sub_postfix(self): check_cmd = ('/usr/lib/nagios/plugins/check_disk -u GB -w 25% -c 20% ' '-K 5% -p /') test_config = { 'filename': '/etc/nagios/nrpe.d/check_disk_root_testing.cfg', 'expected_settings': {'command[check_disk_root]': check_cmd}, 'juju_kv': {'sub_postfix': '_testing'}, 'filedelim': '=', } self.check_nrpe_setting(**test_config) def test_custom_disk_check_params(self): chk_key = 'command[check_disk_root]=/usr/lib/nagios/plugins/check_disk' test_config = { 'filename': '/etc/nagios/nrpe.d/check_disk_root_sub.cfg', 'expected_settings': {chk_key: '-u GB -w 5% -c 1% -K 10% -p /'}, 'juju_kv': {'disk_root': '-u GB -w 5% -c 1% -K 10%'} } self.check_nrpe_setting(**test_config) def test_custom_zombie_check_params(self): chk_key = ('command[check_zombie_procs]=/usr/lib/nagios/plugins/' 'check_procs') test_config = { 'filename': '/etc/nagios/nrpe.d/check_zombie_procs_sub.cfg', 'expected_settings': {chk_key: '-w 6 -c 12 -s Z'}, 'juju_kv': {'zombies': '-w 6 -c 12 -s Z'} } self.check_nrpe_setting(**test_config) def test_custom_procs_check_params(self): chk_key = ('command[check_zombie_procs]=/usr/lib/nagios/plugins/' 'check_procs') test_config = { 'filename': '/etc/nagios/nrpe.d/check_total_procs_sub.cfg', 'expected_settings': {chk_key: '-w 40 -c 60'}, 'juju_kv': {'procs': '-w 40 -c 60'} } self.check_nrpe_setting(**test_config) def test_custom_load_check_params(self): chk_key = 'command[check_load]=/usr/lib/nagios/plugins/check_load' test_config = { 'filename': '/etc/nagios/nrpe.d/check_load_sub.cfg', 'expected_settings': {chk_key: '-w 9,9,9 -c 16,16,16'}, 'juju_kv': {'load': '-w 9,9,9 -c 16,16,16'} } self.check_nrpe_setting(**test_config) def test_custom_users_check_params(self): chk_key = 'command[check_users]=/usr/lib/nagios/plugins/check_users' test_config = { 'filename': '/etc/nagios/nrpe.d/check_users_sub.cfg', 'expected_settings': {chk_key: '-w 40 -c 50'}, 'juju_kv': {'users': '-w 40 -c 50'} } self.check_nrpe_setting(**test_config) def test_custom_swap_check_params(self): chk_key = 'command[check_swap]=/usr/lib/nagios/plugins/check_swap' test_config = { 'filename': '/etc/nagios/nrpe.d/check_swap_sub.cfg', 'expected_settings': {chk_key: '-w 5% -c 1%'}, 'juju_kv': {'swap': '-w 5% -c 1%'} } self.check_nrpe_setting(**test_config) def test_custom_swap_activity_check_params(self): chk_key = 'command[check_swap_activity]=/usr/local/lib/nagios/plugins/check_swap_activity' test_config = { 'filename': '/etc/nagios/nrpe.d/check_swap_activity_sub.cfg', 'expected_settings': {chk_key: '-w 20 -c 700'}, 'juju_kv': {'swap_activity': '-w 20 -c 700'} } self.check_nrpe_setting(**test_config) def test_custom_conntrack_check_params(self): chk_key = ('command[check_conntrack]=/usr/local/lib/nagios/plugins/' 'check_conntrack.sh') test_config = { 'filename': '/etc/nagios/nrpe.d/check_conntrack_sub.cfg', 'expected_settings': {chk_key: '-w 50 -c 70'}, 'juju_kv': {'conntrack': '-w 50 -c 70'} } self.check_nrpe_setting(**test_config) def test_custom_lacp_bonds(self): chk_key = ('command[check_lacp_bond0]=/usr/local/lib/nagios/plugins/' 'check_lacp_bond.py -i bond0') test_config = { 'filename': '/etc/nagios/nrpe.d/check_lacp_bond0.cfg', 'expected_settings': {chk_key: 'bond0'}, 'juju_kv': {'lacp_bonds': 'bond0'} } self.check_nrpe_setting(**test_config) def test_custom_netlinks_string(self): chk_key = ('command[check_netlinks_eth0]=/usr/local/lib/nagios/plugins/' 'check_netlinks.py -i eth0 -m 1500 -s 10000') test_config = { 'filename': '/etc/nagios/nrpe.d/check_netlinks_eth0.cfg', 'expected_settings': {chk_key: 'eth0 mtu:1500 speed:10000'}, 'juju_kv': {'netlinks': 'eth0 mtu:1500 speed:10000'} } self.check_nrpe_setting(**test_config) def test_custom_netlinks_yaml_list(self): chk_key = ('command[check_netlinks_eth0]=/usr/local/lib/nagios/plugins/' 'check_netlinks.py -i eth0 -m 1500 -s 10000') test_config = { 'filename': '/etc/nagios/nrpe.d/check_netlinks_eth0.cfg', 'expected_settings': {chk_key: ['eth0 mtu:1500 speed:10000']}, 'juju_kv': {'netlinks': "['eth0 mtu:1500 speed:10000']"} } self.check_nrpe_setting(**test_config) if __name__ == '__main__': unittest.main()