392 lines
15 KiB
Python
Executable File
392 lines
15 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
import amulet
|
|
import unittest
|
|
import time
|
|
import yaml
|
|
from charmhelpers.contrib.amulet.utils import (
|
|
AmuletUtils,
|
|
)
|
|
autils = AmuletUtils()
|
|
PAUSE_TIME = 30
|
|
|
|
|
|
class TestDeployment(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.deployment = amulet.Deployment(series='trusty')
|
|
cls.deployment.add('mysql')
|
|
cls.deployment.add('nrpe')
|
|
cls.deployment.add('nagios')
|
|
cls.deployment.configure('mysql', {'dataset-size': '10%'})
|
|
cls.deployment.relate('nrpe:monitors',
|
|
'nagios:monitors')
|
|
cls.deployment.relate('nrpe:local-monitors',
|
|
'mysql:local-monitors')
|
|
cls.deployment.expose('nagios')
|
|
try:
|
|
cls.deployment.setup(timeout=900)
|
|
cls.deployment.sentry.wait()
|
|
except amulet.helpers.TimeoutError:
|
|
msg = "Environment wasn't stood up in time"
|
|
amulet.raise_status(amulet.SKIP, msg=msg)
|
|
except Exception:
|
|
raise
|
|
|
|
def check_nrpe_setting(self, filename, expected_settings, juju_kv,
|
|
filedelim=None):
|
|
self.nrpe_sentry = self.deployment.sentry['nrpe'][0]
|
|
if juju_kv:
|
|
self.deployment.configure('nrpe', juju_kv)
|
|
time.sleep(PAUSE_TIME)
|
|
nrpe_contents = self.nrpe_sentry.file_contents(filename)
|
|
for line in nrpe_contents.split('\n'):
|
|
if not line:
|
|
continue
|
|
line_key = line.split(filedelim)[0]
|
|
if line_key in expected_settings.keys():
|
|
line_value = ' '.join(line.split(filedelim)[1:])
|
|
if line_value != expected_settings[line_key]:
|
|
msg = ('Setting %s in %s did not contain expected value '
|
|
'(%s != %s)') % (line_key, filename, line_value,
|
|
expected_settings[line_key])
|
|
amulet.raise_status(amulet.FAIL, msg=msg)
|
|
|
|
def test_monitors_relation_sub_monitors(self):
|
|
""" Check default monitor definitions are passed to Nagios """
|
|
self.deployment.configure('nrpe', {'monitors': ''})
|
|
time.sleep(PAUSE_TIME)
|
|
self.nrpe_sentry = self.deployment.sentry['nrpe'][0]
|
|
relation_data = self.nrpe_sentry.relation(
|
|
'monitors',
|
|
'nagios:monitors',
|
|
)
|
|
monitors = yaml.safe_load(relation_data['monitors'])
|
|
checks = [
|
|
'check_mem_sub',
|
|
'check_disk_root_sub',
|
|
'check_swap_sub',
|
|
'check_load_sub',
|
|
'check_users_sub',
|
|
'check_zombie_procs_sub',
|
|
'check_total_procs_sub',
|
|
'check_conntrack_sub',
|
|
'check_swap_activity_sub',
|
|
]
|
|
for check in checks:
|
|
if check not in monitors['monitors']['remote']['nrpe'].keys():
|
|
amulet.raise_status(
|
|
amulet.FAIL,
|
|
msg='{} not found in monitors relation'.format(check)
|
|
)
|
|
|
|
def test_monitors_relation_principal_monitors(self):
|
|
""" Check monitor definitions from principal are passed to Nagios """
|
|
self.nrpe_sentry = self.deployment.sentry['nrpe'][0]
|
|
relation_data = self.nrpe_sentry.relation(
|
|
'monitors',
|
|
'nagios:monitors',
|
|
)
|
|
monitors = yaml.safe_load(relation_data['monitors'])
|
|
if 'mysql' not in monitors['monitors']['remote'].keys():
|
|
amulet.raise_status(
|
|
amulet.FAIL,
|
|
msg='mysql remote monitor not found in monitors relation',
|
|
)
|
|
nrpe_checks = monitors['monitors']['remote']['nrpe'].keys()
|
|
if 'check_proc_mysqld_principal' not in nrpe_checks:
|
|
amulet.raise_status(
|
|
amulet.FAIL,
|
|
msg='mysql process monitor not found in monitors relation',
|
|
)
|
|
|
|
def test_monitors_relation_user_monitors(self):
|
|
""" Check user configured monitor definitions are passed to Nagios """
|
|
user_monitors = {
|
|
'version': '0.3',
|
|
'monitors': {
|
|
'local': {
|
|
'procrunning': {
|
|
'rsync': {
|
|
'max': 1,
|
|
'executable': 'rsync',
|
|
'name': 'RSYNc Running',
|
|
'min': 1
|
|
},
|
|
'jujud': {
|
|
'max': 1,
|
|
'executable': 'jujud',
|
|
'name': 'Juju Running',
|
|
'min': 1
|
|
}
|
|
}
|
|
},
|
|
'remote': {
|
|
'tcp': {
|
|
'ssh': {
|
|
'warning': 2,
|
|
'critical': 10,
|
|
'name': 'SSH Running',
|
|
'timeout': 12,
|
|
'port': 22,
|
|
'string': 'SSH.*',
|
|
'expect': None
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
self.deployment.configure(
|
|
'nrpe', {'monitors': yaml.dump(user_monitors)}
|
|
)
|
|
time.sleep(PAUSE_TIME)
|
|
self.nrpe_sentry = self.deployment.sentry['nrpe'][0]
|
|
relation_data = self.nrpe_sentry.relation(
|
|
'monitors',
|
|
'nagios:monitors',
|
|
)
|
|
monitors = yaml.safe_load(relation_data['monitors'])
|
|
checks = ['check_proc_jujud_user', 'check_proc_jujud_user']
|
|
for check in checks:
|
|
if check not in monitors['monitors']['remote']['nrpe'].keys():
|
|
amulet.raise_status(
|
|
amulet.FAIL,
|
|
msg='{} not found in monitors relation'.format(check),
|
|
)
|
|
if 'ssh' not in monitors['monitors']['remote']['tcp'].keys():
|
|
amulet.raise_status(
|
|
amulet.FAIL,
|
|
msg='{} not found in monitors relation'.format(check),
|
|
)
|
|
|
|
def test_services(self):
|
|
""" Test basic services are running """
|
|
self.nagios_sentry = self.deployment.sentry['nagios'][0]
|
|
self.nrpe_sentry = self.deployment.sentry['nrpe'][0]
|
|
commands = {
|
|
self.nrpe_sentry: ['service nagios-nrpe-server status'],
|
|
self.nagios_sentry: ['service nagios3 status'],
|
|
}
|
|
ret = autils.validate_services(commands)
|
|
if ret:
|
|
amulet.raise_status(amulet.FAIL, msg=ret)
|
|
|
|
def test_config_nagios_master(self):
|
|
unit = self.deployment.sentry['nagios'][0]
|
|
nagios_relation = unit.relation('monitors', 'nrpe:monitors')
|
|
ipaddr = nagios_relation['private-address']
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.cfg',
|
|
'expected_settings': {
|
|
'allowed_hosts': '127.0.0.1,10.0.0.10,' + ipaddr
|
|
},
|
|
'juju_kv': {'nagios_master': '10.0.0.10'},
|
|
'filedelim': '=',
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_config_rsync_fragment(self):
|
|
test_config = {
|
|
'filename': '/etc/rsync-juju.d/010-nrpe-external-master.conf',
|
|
'expected_settings': {'hosts allow': '10.0.0.10'},
|
|
'juju_kv': {
|
|
'nagios_master': '10.0.0.10',
|
|
'export_nagios_definitions': True,
|
|
},
|
|
'filedelim': '=',
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_config_rsync(self):
|
|
test_config = {
|
|
'filename': '/etc/rsyncd.conf',
|
|
'expected_settings': {'&include': '/etc/rsync-juju.d'},
|
|
'juju_kv': {'export_nagios_definitions': True},
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_config_server_port(self):
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.cfg',
|
|
'expected_settings': {'server_port': '5888'},
|
|
'juju_kv': {'server_port': '5888'},
|
|
'filedelim': '=',
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
self.deployment.configure('nrpe', {'server_port': '5888'})
|
|
|
|
def test_config_debug(self):
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.cfg',
|
|
'expected_settings': {'debug': '1'},
|
|
'juju_kv': {'debug': 'True'},
|
|
'filedelim': '=',
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
self.deployment.configure('nrpe', {'debug': 'True'})
|
|
|
|
def test_config_dont_blame_nrpe(self):
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.cfg',
|
|
'expected_settings': {'dont_blame_nrpe': '1'},
|
|
'juju_kv': {'dont_blame_nrpe': 'True'},
|
|
'filedelim': '=',
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
self.deployment.configure('nrpe', {'dont_blame_nrpe': 'True'})
|
|
|
|
def test_nagios_host_context(self):
|
|
hostname = 'bob-mysql-0'
|
|
test_config = {
|
|
'filename': '/var/lib/nagios/export/host__%s.cfg' % (hostname),
|
|
'expected_settings': {'host_name': hostname,
|
|
'use': 'masterhostgroup',
|
|
'hostgroups': 'machines, myhostgroup1'},
|
|
'juju_kv': {
|
|
'nagios_host_context': 'bob',
|
|
'hostcheck_inherit': 'masterhostgroup',
|
|
'nagios_hostname_type': 'unit',
|
|
'hostgroups': 'myhostgroup1',
|
|
'export_nagios_definitions': True,
|
|
},
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_nagios_hostname_type(self):
|
|
sentry = self.deployment.sentry['nrpe'][0]
|
|
hostname = sentry.run('hostname')[0]
|
|
test_config = {
|
|
'filename': '/var/lib/nagios/export/host__%s.cfg' % (hostname),
|
|
'expected_settings': {'host_name': hostname,
|
|
'use': 'masterhostgroup',
|
|
'hostgroups': 'machines, myhostgroup1'},
|
|
'juju_kv': {
|
|
'nagios_host_context': 'bob',
|
|
'hostcheck_inherit': 'masterhostgroup',
|
|
'hostgroups': 'myhostgroup1',
|
|
'nagios_hostname_type': 'host',
|
|
'export_nagios_definitions': True
|
|
},
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_sub_postfix(self):
|
|
check_cmd = ('/usr/lib/nagios/plugins/check_disk -u GB -w 25% -c 20% '
|
|
'-K 5% -p /')
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_disk_root_testing.cfg',
|
|
'expected_settings': {'command[check_disk_root]': check_cmd},
|
|
'juju_kv': {'sub_postfix': '_testing'},
|
|
'filedelim': '=',
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_disk_check_params(self):
|
|
chk_key = 'command[check_disk_root]=/usr/lib/nagios/plugins/check_disk'
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_disk_root_sub.cfg',
|
|
'expected_settings': {chk_key: '-u GB -w 5% -c 1% -K 10% -p /'},
|
|
'juju_kv': {'disk_root': '-u GB -w 5% -c 1% -K 10%'}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_zombie_check_params(self):
|
|
chk_key = ('command[check_zombie_procs]=/usr/lib/nagios/plugins/'
|
|
'check_procs')
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_zombie_procs_sub.cfg',
|
|
'expected_settings': {chk_key: '-w 6 -c 12 -s Z'},
|
|
'juju_kv': {'zombies': '-w 6 -c 12 -s Z'}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_procs_check_params(self):
|
|
chk_key = ('command[check_zombie_procs]=/usr/lib/nagios/plugins/'
|
|
'check_procs')
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_total_procs_sub.cfg',
|
|
'expected_settings': {chk_key: '-w 40 -c 60'},
|
|
'juju_kv': {'procs': '-w 40 -c 60'}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_load_check_params(self):
|
|
chk_key = 'command[check_load]=/usr/lib/nagios/plugins/check_load'
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_load_sub.cfg',
|
|
'expected_settings': {chk_key: '-w 9,9,9 -c 16,16,16'},
|
|
'juju_kv': {'load': '-w 9,9,9 -c 16,16,16'}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_users_check_params(self):
|
|
chk_key = 'command[check_users]=/usr/lib/nagios/plugins/check_users'
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_users_sub.cfg',
|
|
'expected_settings': {chk_key: '-w 40 -c 50'},
|
|
'juju_kv': {'users': '-w 40 -c 50'}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_swap_check_params(self):
|
|
chk_key = 'command[check_swap]=/usr/lib/nagios/plugins/check_swap'
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_swap_sub.cfg',
|
|
'expected_settings': {chk_key: '-w 5% -c 1%'},
|
|
'juju_kv': {'swap': '-w 5% -c 1%'}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_swap_activity_check_params(self):
|
|
chk_key = 'command[check_swap_activity]=/usr/local/lib/nagios/plugins/check_swap_activity'
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_swap_activity_sub.cfg',
|
|
'expected_settings': {chk_key: '-w 20 -c 700'},
|
|
'juju_kv': {'swap_activity': '-w 20 -c 700'}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_conntrack_check_params(self):
|
|
chk_key = ('command[check_conntrack]=/usr/local/lib/nagios/plugins/'
|
|
'check_conntrack.sh')
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_conntrack_sub.cfg',
|
|
'expected_settings': {chk_key: '-w 50 -c 70'},
|
|
'juju_kv': {'conntrack': '-w 50 -c 70'}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_lacp_bonds(self):
|
|
chk_key = ('command[check_lacp_bond0]=/usr/local/lib/nagios/plugins/'
|
|
'check_lacp_bond.py -i bond0')
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_lacp_bond0.cfg',
|
|
'expected_settings': {chk_key: 'bond0'},
|
|
'juju_kv': {'lacp_bonds': 'bond0'}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_netlinks_string(self):
|
|
chk_key = ('command[check_netlinks_eth0]=/usr/local/lib/nagios/plugins/'
|
|
'check_netlinks.py -i eth0 -m 1500 -s 10000')
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_netlinks_eth0.cfg',
|
|
'expected_settings': {chk_key: 'eth0 mtu:1500 speed:10000'},
|
|
'juju_kv': {'netlinks': 'eth0 mtu:1500 speed:10000'}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
def test_custom_netlinks_yaml_list(self):
|
|
chk_key = ('command[check_netlinks_eth0]=/usr/local/lib/nagios/plugins/'
|
|
'check_netlinks.py -i eth0 -m 1500 -s 10000')
|
|
test_config = {
|
|
'filename': '/etc/nagios/nrpe.d/check_netlinks_eth0.cfg',
|
|
'expected_settings': {chk_key: ['eth0 mtu:1500 speed:10000']},
|
|
'juju_kv': {'netlinks': "['eth0 mtu:1500 speed:10000']"}
|
|
}
|
|
self.check_nrpe_setting(**test_config)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|