From 1f09875aa3421173d18a9f7846f3e2d23e63e975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=92=D0=BB=D0=B0=D0=B4?= =?UTF-8?q?=D0=B8=D0=BC=D0=B8=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 7 Feb 2017 01:39:30 +0300 Subject: [PATCH] Update russian docs. Add compatible for version 0.7. Add config module. Fix discovery module. Update test.py --- AUTHORS | 1 + build/runtests.sh | 2 +- build/tests.py | 426 +++++++++++++++++++++++++++------ conf.d/marathon_check.json | 12 + conf.d/self_check.json | 17 ++ conf.d/selfcheck.json | 12 - conf/surok.json | 8 +- debian/install | 4 + docs/README.md | 1 + docs/ru/surok.configuration.md | 99 ++++++-- docs/ru/templates.md | 137 ++++++++++- surok.py | 54 +---- surok.spec | 10 +- surok/config.py | 410 +++++++++++++++++++++++++++++++ surok/discovery.py | 385 +++++++++++++---------------- surok/logger.py | 97 +++++--- surok/system.py | 4 +- surok_fedora.spec | 8 +- 18 files changed, 1267 insertions(+), 420 deletions(-) mode change 100644 => 100755 build/tests.py create mode 100644 conf.d/marathon_check.json create mode 100644 conf.d/self_check.json delete mode 100644 conf.d/selfcheck.json create mode 100644 surok/config.py diff --git a/AUTHORS b/AUTHORS index a720639..354c3b8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,2 +1,3 @@ Denis Zheleztsov Denis Ryabyy +Evgeniy Vasilev diff --git a/build/runtests.sh b/build/runtests.sh index 37cbbe7..f54968f 100755 --- a/build/runtests.sh +++ b/build/runtests.sh @@ -5,7 +5,7 @@ set -e . functions.sh function run_tests() { - docker run -ti -v $(pwd)/tests.py:/opt/surok/tests.py \ + docker run --rm -ti -v $(pwd)/tests.py:/opt/surok/tests.py \ -v $(pwd)/tests_entrypoint.sh:/tests_entrypoint.sh \ --entrypoint /tests_entrypoint.sh \ surok_base:latest diff --git a/build/tests.py b/build/tests.py old mode 100644 new mode 100755 index 3828fe4..b8d1dd0 --- a/build/tests.py +++ b/build/tests.py @@ -1,80 +1,372 @@ +#!/usr/bin/python3 import unittest import json import os import re +import sys +import surok.config +import surok.logger +import surok.discovery +import hashlib +from surok.config import Config -class TestLoadConfig(unittest.TestCase): +class Logger(surok.logger.Logger): + _out='' + _err='' + def _log2err(self,out): + self._err+=out - def test_main_conf(self): - # Load base configurations - surok_conf = '/etc/surok/conf/surok.json' - # Read config file - f = open(surok_conf, 'r') - conf = json.loads(f.read()) - f.close() + def _log2out(self,out): + self._out+=out - self.assertIn('confd', conf) - self.assertTrue(os.path.isdir(conf['confd'])) - self.assertIn('wait_time', conf) - self.assertIn('lock_dir', conf) - self.assertTrue(os.path.isdir(conf['lock_dir'])) + def geterr(self): + return self._err + + def getout(self): + return self._out + + def reset(self): + self._err='' + self._out='' + +class DiscoveryTestingTemplate: + _testing={} + _testing_fqdn_a={ + "test.zzz0.test":['10.0.0.1','10.1.0.1'], + "test.zzz1.test":['10.0.1.1','10.1.1.1'], + "test.zzz2.test":['10.0.2.1','10.1.2.1'], + "test.zzz3.test":['10.0.3.1','10.1.3.1'] + } + _testing_fqdn_srv={} + + def __init__(self): + super().__init__() + self._orig_logger=surok.logger.Logger() + + def do_query_a(self,fqdn): + res=self._testing_fqdn_a.get(fqdn,[]) + if res: + return res + else: + self._orig_logger.error('Testing FQDN '+fqdn+' not found in test A records') + sys.exit(2) + + def do_query_srv(self,fqdn): + res=self._testing_fqdn_srv.get(fqdn,[]) + if res or fqdn.startswith('_tname_e.') or fqdn.find('._udp.'): + return res + else: + self._orig_logger.error('Testing FQDN '+fqdn+' not found in test SRV records') + sys.exit(2) + + def update_data(self): + class_name=self.__class__.__name__ + tgen={ + "name": ["zzz0","zzy0","zzy1","zzz1"], + "host": ["test.zzz0.test","test.zzz1.test","test.zzz2.test","test.zzz3.test"], + "serv": ["tname_aa","tname_ab","tname_ba","tname_bb"], + "ports": [12341,12342,12343,12344], + "servicePorts": [21221,21222,21223,21224] + } + if self._testing.get(class_name,True): + if class_name == 'DiscoveryMarathon': + _tasks=[] + _ports={} + for id in (0,1,2,3): + ports=[]+tgen['ports'] + servicePorts=[]+tgen['servicePorts'] + appId='/'.join(str(tgen['name'][id]+'.xxx.yyy.').split('.')[::-1]) + _ports[appId]=[] + for pid in (0,1,2,3): + ports[pid]+=pid*10 + servicePorts[pid]+=pid*100 + for prot in ['tcp','udp']: + if pid<2 or prot == 'tcp': + _ports[appId].append({'containerPort': 0, + 'hostPort': 0, + 'labels': {}, + 'name': tgen['serv'][pid], + 'protocol': prot, + 'servicePort': servicePorts[pid]}) + + _tasks.append({'appId':appId, + 'host':tgen['host'][id], + 'ports':ports, + 'servicePorts':servicePorts}) + #_tname_a._zzy0.yyy.xxx._tcp.marathon.mesos + self._tasks=_tasks + self._ports=_ports + elif class_name == 'DiscoveryMesos': + for id in (0,1,2,3): + ports=[]+tgen['ports'] + for pid in (0,1,2,3): + ports[pid]+=pid*10 + for prot in ['tcp','udp']: + if pid<2 or prot == 'tcp': + for fqdn in ['_'+tgen['serv'][pid]+'._'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain'), + '_'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain')]: + if not self._testing_fqdn_srv.get(fqdn): + self._testing_fqdn_srv[fqdn]=[] + self._testing_fqdn_srv[fqdn].append({'name':tgen['host'][id],'port':ports[pid]}) + + self._testing[class_name]=False + +class DiscoveryMesos(DiscoveryTestingTemplate,surok.discovery.DiscoveryMesos): + pass + +class DiscoveryMarathon(DiscoveryTestingTemplate,surok.discovery.DiscoveryMarathon): + pass + +class Discovery(surok.discovery.Discovery): + _discoveries={} + def __init__(self): + self._config=Config() + self._logger=Logger() + if not self._discoveries.get('mesos_dns'): + self._discoveries['mesos_dns']=DiscoveryMesos() + if not self._discoveries.get('marathon_api'): + self._discoveries['marathon_api']=DiscoveryMarathon() + +class Test01_Logger(unittest.TestCase): + def test_01_logger_default_level(self): + logger = Logger() + self.assertEqual(logger.get_level(), 'info') + + def test_02_logger_output_levels(self): + message='log message' + tests={ + 'debug':{ + 'assertIn':['ERROR: {}','WARNING: {}','INFO: {}','DEBUG: {}'], + 'assertNotIn':[] + }, + 'info':{ + 'assertIn':['ERROR: {}','WARNING: {}','INFO: {}'], + 'assertNotIn':['DEBUG: {}'] + }, + 'warning':{ + 'assertIn':['ERROR: {}','WARNING: {}'], + 'assertNotIn':['INFO: {}','DEBUG: {}'] + }, + 'error':{ + 'assertIn':['ERROR: {}'], + 'assertNotIn':['WARNING: {}','INFO: {}','DEBUG: {}'] + } + } + logger = Logger() + for value01 in tests.keys(): + logger.reset() + logger.set_level(value01) + logger.error(message) + logger.warning(message) + logger.info(message) + logger.debug(message) + resmessage=logger.geterr()+logger.getout() + for test_name in tests[value01].keys(): + for test_value in tests[value01][test_name]: + with self.subTest(msg='Testing Logger for ...', loglevel=value01): + test_message=test_value.format(message) + eval('self.{}(test_message,resmessage)'.format(test_name)) + +class Test02_LoadConfig(unittest.TestCase): + + def test_01_default_values(self): + config=Config() + with self.subTest(msg='Testing default values for Config.', dump=config.dump()): + self.assertEqual(config.get('confd'), '/etc/surok/conf.d') + self.assertEqual(config.get('default_discovery'), 'mesos_dns') + self.assertEqual(config.get('lock_dir'), '/var/tmp') + self.assertEqual(config.get('loglevel'), 'info') + self.assertEqual(dict(config.get('marathon',{})).get('enabled'), False) + self.assertEqual(dict(config.get('mesos',{})).get('enabled'), False) + self.assertEqual(dict(config.get('memcached',{})).get('enabled'), False) + self.assertEqual(config.get('version'), '0.7') + self.assertEqual(config.get('wait_time'), 20) + + def test_02_main_conf(self): + config=Config('/etc/surok/conf/surok.json') + with self.subTest(msg='Testing load config for Config.', dump=config.dump()): + self.assertEqual(config.hash(), '545c20b322a6ba5fef9c7d2416d80178f26a924b') + + def test_03_apps_conf(self): + tests=[ + { + 'env':{}, + 'self_check.json':'a4e109b9fec696776fd3df091b607e9c1489748c', + 'marathon_check.json':'6be7f26d421d4a0a2e7b089184be0c0e3a50f986' + }, + { + 'env':{'SUROK_DISCOVERY_GROUP':'xxx.yyy'}, + 'self_check.json':'38ab770ff2ba69bf70673288425337ff3c18a807', + 'marathon_check.json':'7b0cb4eab2d8e0f901cc567df28b17279af21baa' + }, + { + 'env':{'MARATHON_APP_ID':'/xxx/yyy/zzz'}, + 'self_check.json':'cbd2a15179649d0e06f98bd64e024481a944d65c', + 'marathon_check.json':'08a382d14285feb1f22b92ba597ecf73d654a2e0' + } + ] + config=Config() + for test in tests: + config.set('env',test['env']) + config.update_apps() + for app in config.apps: + with self.subTest(msg='Testing AppConfig for ...', env=test['env'], conf_name=app.get('conf_name'), dump=app.dump()): + self.assertEqual(test[app.get('conf_name')],app.hash()) + + def test_04_apps_conf(self): + tests={ + 'confd':{ + 'assertEqual': ['/var', '/var/tmp', '/etc/surok/conf.d'], + 'assertNotEqual': [20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True] + }, + 'default_discovery':{ + 'assertEqual':['marathon_api', 'mesos_dns'], + 'assertNotEqual':[20, 'test', None] + }, + 'lock_dir':{ + 'assertEqual':['/var', '/etc/surok/conf.d', '/var/tmp'], + 'assertNotEqual':[20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True] + }, + 'loglevel':{ + 'assertEqual':['error', 'debug', 'info', 'warning'], + 'assertNotEqual':['errrr', 'DEBUG','warn', 'test', 1, None, True] + }, + 'version':{ + 'assertEqual': ['0.7', '0.8'], + 'assertNotEqual': ['0,7', '07', '0.9', 0.7, 0.8, None] + }, + 'wait_time':{ + 'assertEqual': [10, 15, 20], + 'assertNotEqual': ['10', '15', None, True] + } + + } + config=Config() + for name01 in tests.keys(): + oldvalue=config.get(name01) + for test_name in tests[name01].keys(): + for value01 in tests[name01][test_name]: + config.set_config({name01:value01}) + test_value=config.get(name01) + with self.subTest(msg='Testing Config Change for values...', name=name01, value=value01, test_value=test_value): + eval('self.{}(value01, test_value)'.format(test_name)) + config.set(name01,oldvalue) -class TestLogger(unittest.TestCase): - - def test_debug(self): - from surok.logger import Logger - m = Logger() - self.assertIn('DEBUG', m.testing('debug','log message')) - - def test_info(self): - from surok.logger import Logger - m = Logger() - self.assertIn('INFO', m.testing('info','log message')) - - def test_warning(self): - from surok.logger import Logger - m = Logger() - self.assertIn('WARNING', m.testing('warning','log message')) - - def test_error(self): - from surok.logger import Logger - m = Logger() - self.assertIn('ERROR', m.testing('error','log message')) - - -class TestMemcachedDiscovery(unittest.TestCase): - - def test_discovery_memcache(self): - from surok.system import discovery_memcached - from surok.discovery import Discovery - # Load base configurations - surok_conf = '/etc/surok/conf/surok.json' - # Read config file - f = open(surok_conf, 'r') - conf = json.loads(f.read()) - f.close() - d=Discovery(conf) - self.assertEqual(discovery_memcached(conf), []) - - -class TestGetGroup(unittest.TestCase): - - def test_get_group_from_service(self): - from surok.discovery import DiscoveryTemplate - d=DiscoveryTemplate({}) - self.assertEqual('xxx.yyy.zzz',d.get_group({'group':'xxx.yyy.zzz'}, {})) - - def test_get_group_from_env(self): - from surok.discovery import DiscoveryTemplate - d=DiscoveryTemplate({}) - self.assertEqual('xxx.yyy.zzz',d.get_group({}, {'env':{'SUROK_DISCOVERY_GROUP':'xxx.yyy.zzz'}})) - - def test_get_group_from_marathon_id(self): - from surok.discovery import DiscoveryTemplate - d=DiscoveryTemplate({}) - self.assertEqual('xxx.yyy.zzz',d.get_group({}, {'env':{'MARATHON_APP_ID':'/zzz/yyy/xxx/www'}})) +class Test03_Discovery(unittest.TestCase): + def test_01_discovery(self): + tests={ + 'T':{ #mesos_enabled + 'T':{ #marathon_enabled + '0.7':{ #version + 'mesos_dns':{ #default_discovery + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', #app['conf_name'] + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + }, + 'marathon_api':{ + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + }, + 'marathon_api':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + } + } + }, + 'F':{ + '0.7':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + } + } + }, + 'F':{ + 'T':{ + '0.7':{ + 'mesos_dns':{ + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + } + } + }, + 'F':{ + '0.7':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + } + } + } + } + config=Config('/etc/surok/conf/surok.json') + config.set('env',{'SUROK_DISCOVERY_GROUP':'xxx.yyy'}) + discovery=Discovery() + for mesos_enabled in tests.keys(): + for marathon_enabled in tests[mesos_enabled].keys(): + for version in tests[mesos_enabled][marathon_enabled].keys(): + for default_discovery in tests[mesos_enabled][marathon_enabled][version].keys(): + config.set_config({'default_discovery':default_discovery, + 'mesos':{'enabled':(mesos_enabled=='T')}, + 'marathon':{'enabled':(marathon_enabled=='T')}, + 'version':version}) + discovery.update_data() + for app in config.apps: + conf_name=app.get('conf_name') + with self.subTest(msg='Testing Discovery for values...', config=config.dump(), conf_name=conf_name): + self.assertEqual(hashlib.sha1(json.dumps(discovery.resolve(app), sort_keys=True).encode()).hexdigest(), + tests[mesos_enabled][marathon_enabled][version][default_discovery][conf_name]) if __name__ == '__main__': unittest.main() diff --git a/conf.d/marathon_check.json b/conf.d/marathon_check.json new file mode 100644 index 0000000..15e784f --- /dev/null +++ b/conf.d/marathon_check.json @@ -0,0 +1,12 @@ +{ + "services": [ + { + "name": "zzy*", + "ports": ["tname_a*"] + } + ], + "template": "templates/selfcheck.jj2", + "dest": "selfcheck", + "reload_cmd": "/bin/echo selfcheck ok", + "discovery": "marathon_api" +} diff --git a/conf.d/self_check.json b/conf.d/self_check.json new file mode 100644 index 0000000..4ce08e7 --- /dev/null +++ b/conf.d/self_check.json @@ -0,0 +1,17 @@ +{ + "services": [ + {"name":"zzy0", + "ports":["tname_aa","tname_ab","tname_ba","tname_bb","tname_d"] + }, + {"name":"zzy1", + "ports":["tname_aa","tname_ab","tname_ba","tname_bb"] + }, + {"name":"zzz0", + "ports":["tname_aa","tname_bb"] + }, + {"name":"zzz1"} + ], + "template": "templates/selfcheck.jj2", + "dest": "selfcheck", + "reload_cmd": "/bin/echo selfcheck ok" +} diff --git a/conf.d/selfcheck.json b/conf.d/selfcheck.json deleted file mode 100644 index 56cec93..0000000 --- a/conf.d/selfcheck.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "services": [ - { - "name": "", - "group": "mesos" - } - ], - "conf_name": "selfcheck", - "template": "templates/selfcheck.jj2", - "dest": "selfcheck", - "reload_cmd": "/bin/echo selfcheck ok" -} diff --git a/conf/surok.json b/conf/surok.json index e1c58f2..20bd0fd 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,16 +1,13 @@ { + "version":"0.8", "marathon": { "enabled": false, "restart": false, "force": true, "host": "http://marathon.mesos:8080" }, - "consul": { - "enabled": false, - "domain": "service.dc1.consul" - }, "mesos":{ - "enabled": true, + "enabled": false, "domain": "marathon.mesos" }, "default_discovery": "mesos_dns", @@ -18,7 +15,6 @@ "wait_time": 20, "lock_dir": "/var/tmp", "loglevel": "info", - "container": false, "memcached": { "enabled": false, "discovery": { diff --git a/debian/install b/debian/install index 74eab76..d00a88b 100644 --- a/debian/install +++ b/debian/install @@ -1,7 +1,11 @@ conf/surok.json etc/surok/conf +conf.d/self_check.json etc/surok/conf.d +conf.d/marathon_check.json etc/surok/conf.d +templates/selfcheck.jj2 etc/surok/templates surok/templates.py opt/surok/surok surok/system.py opt/surok/surok surok/logger.py opt/surok/surok surok/__init__.py opt/surok/surok surok/discovery.py opt/surok/surok +surok/config.py opt/surok/surok surok.py opt/surok diff --git a/docs/README.md b/docs/README.md index 82774d8..a5815d2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,6 +31,7 @@ Service discovery for Apache Mesos. * Denis Zheleztsov * Denis Ryabyy +* Evgeniy Vasilev ## LICENSE diff --git a/docs/ru/surok.configuration.md b/docs/ru/surok.configuration.md index 213ae15..754bc5d 100644 --- a/docs/ru/surok.configuration.md +++ b/docs/ru/surok.configuration.md @@ -1,30 +1,89 @@ -# Конфигурация Surok (0.7.x) - -**/etc/surok/conf/surok.json** -Разберем конфигурационный файл по опциям +# Конфигурация Surok (0.8.x) +**/etc/surok/conf/surok.json** Разберем конфигурационный файл по опциям ``` { + "version": "0.8" "marathon": { - "force": true, - "host": "marathon.mesos:8080", - "enabled": true + "enabled": false, + "restart": false, + "force": true, + "host": "http://marathon.mesos:8080" }, + "consul": { + "enabled": false, + "domain": "service.dc1.consul" + }, + "mesos":{ + "enabled": true, + "domain": "marathon.mesos" + }, + "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", - "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", - "loglevel": "info|debug" - "container": true|false + "loglevel": "info", + "memcached": { + "enabled": false, + "discovery": { + "enabled": false, + "service": "memcached", + "group": "system" + }, + "hosts": ["localhost:11211"] + } } ``` +## Опции файла конфигурации +* **version** - *string. Не обязательный. По умолчанию "0.7".* +Версия файлов конфигурации, шаблонов. На текущий момент может принимать значения "0.7" или "0.8". + * значение "0.7" - файлы конфигурации версии 0.7.х и более ранних + * значение "0.8" - файлы конфигурации версии 0.8 + +##### версия 0.8 +* **marathon**, **mesos**, **consul**, **memcached** - *dict/hash. Не обязательный. По умолчанию '{"enable":false}'.* +Системы с которыми работает сурок. Если система выключена, то параметры системы и их наличие уже не важны. + * **enable** - *boolean. Не обязательный. По умолчанию false.* + Доступность системы для использования. + + специфичные параметры: + * для Marathon API "marathon" + * **force** - *boolean. Не обязательный. По умолчанию true.* + Рестарт контейнера с force или нет. + * **restart** - *boolean. Не обязательный. По умолчанию false.* + Вкл/выкл. рестарта контейнера + * **host** - *string. Не обязательный. По умолчанию "http://marathon.mesos:8080".* + Адрес Marathon. + * для Consul "consul" + * **domain** - *string. Обязательный.* + Приватный домен Consul + * для mesos DNS "mesos" + * **domain** - *string. Не обязательный. По умолчанию "marathon.mesos".* + Приватный домен Mesos DNS + * для Memcached "memcached" + * **hosts** - + * **discovery** - + * **enabled** - + * **service** - + * **group** - +* **default_discovery** - *string. Не обязательный. По умолчанию "mesos_dns".* + + Может принимать значения: + * "mesos_dns" - Mesos DNS + * "marathon_api"- Marathon API + * "consul_dns" - Consul +* **confd** - *strig. Обязательный.* + Абсолютный путь до директории с конфигурационными файлами приложений. +* **wait_time** - *int. Обязательный.* + Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов. +* **lock_dir** - *string. Обязательный.* + Абсолютный путь до директории с lock-конфигурациями. +* **loglevel** - *string. Не обязательный. По умолчанию "info".* + Уровень логирования. Может принимать значения: "debug", "info", "warning", "error" + +##### версия 0.7 и более ранние +Особенности для файла конфигурации +* **marathon** + * **enabled** - boolean. Вкл/выкл. рестарта контейнера. В версии 0.8 переименована в "restart". +* **domain** - string. Приватный домен Mesos DNS. В версии 0.8 перемещен в dict "mesos". + Обнаружение Mesos DNS включено всегда. -* marathon(v0.7) - hash. В текущей версии отвечает за перезапуск контейнера. Обнаружение сервисов через Marathon пока недоступно. - 1. force - boolean. Рестарт контейнера с force или нет. - 2. host - string. Адрес Marathon. - 3. enabled - boolean. Вкл/выкл. -* confd - strig. Абсолютный путь до директории с конфигурационными файлами приложений. -* domain - string. Домен, который обслуживает mesos-dns. -* wait_time - int. Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов. -* lock_dir - string. Абсолютный путь до директории с lock-конфигурациями. -* loglevel - string. Уровень логирования. -* container(v0.6) - boolean. Определяем внутри или нет контейнера запущен сурок. Меняется логика работы. diff --git a/docs/ru/templates.md b/docs/ru/templates.md index 4b98f32..c7609a5 100644 --- a/docs/ru/templates.md +++ b/docs/ru/templates.md @@ -4,6 +4,127 @@ ## Словарь my в шаблоне +### Версия 0.8 +Surok заполняет словарь my и передает его в шаблон. +``` +{ + "services": { + "asterisk": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1", + "11.0.0.1" + ], + "tcp": { + "rpc":31200, + "web":31201, + "sip":32000 + }, + "udp": { + "sip":31201 + } + }, + { + "name": "nginx.testing-kl123-s1.marathon.mesos.", + "ip": [ + "10.0.0.2", + "11.0.0.2" + ], + "tcp": { + "rpc":31210, + "web":31211, + "sip":32010 + }, + "udp": { + "sip":31211 + } + } + ], + "email": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1" + ], + "tcp": { + "smtp":31200, + "pop":31201 + } + } + ], + "anyport": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1" + ], + "tcp": [ + 31200, + 31201 + ] + } + ] + "env": { + "HOME": "/var/lib/nginx" + } +} +``` + +## Пример реального шаблона + +``` +upstream matrix-http { + hash $remote_addr; +{% for server in my['services']['matrix'] %} + server {{server['name']}}:{{server['tcp']['http']}} max_fails=3; +{% endfor %} +} + +upstream riot-http { + hash $remote_addr; +{% for server in my['services']['riot'] %} + server {{server['name']}}:{{server['tcp'][0]}} max_fails=3; +{% endfor %} +} + +server { + listen 10.15.56.157:80; + server_name matrix.example.com; + + client_max_body_size 10m; + + location / { + proxy_pass http://riot-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /_matrix/ { + proxy_pass http://matrix-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + +} +``` +Так для upstream matrix-http используются именованные порты, а для riot-http – нет. + +## Проверки в шаблоне + +Переменная _my['env']_ является классом python _os.environ_, что позваоляет нам строить различные проверки, например: + +``` +{% if my['env'].get('DB_HOST') -%} +host = '{{my['env']['DB_HOST']}}' +{% else -%} +host = 'localhost' +{% endif -%} +``` + +### Версия 0.7 Surok заполняет словарь my и передает его в шаблон. ``` { @@ -29,13 +150,13 @@ Surok заполняет словарь my и передает его в шаб } ], "service-with-defined-ports": { - "web": [ + "name-of-port0": [ { "name": "f.q.d.n", "port": 12341 } ], - "rpc": [ + "name-of-port2": [ { "name": "f.q.d.n", "port": 12342 @@ -58,34 +179,34 @@ upstream matrix-http { server {{server['name']}}:{{server['port']}} max_fails=3; {% endfor %} } - + upstream riot-http { hash $remote_addr; {% for server in my['services']['riot'] %} server {{server['name']}}:{{server['port']}} max_fails=3; {% endfor %} } - + server { listen 10.15.56.157:80; server_name matrix.example.com; - + client_max_body_size 10m; - + location / { proxy_pass http://riot-http; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - + location /_matrix/ { proxy_pass http://matrix-http; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - + } ``` Так для upstream matrix-http используются именованные порты, а для riot-http – нет. diff --git a/surok.py b/surok.py index 4429e19..e9835f2 100755 --- a/surok.py +++ b/surok.py @@ -10,45 +10,17 @@ from surok.templates import gen from surok.discovery import Discovery from surok.system import reload_conf from surok.logger import Logger +from surok.config import Config logger=Logger() -# Load base configurations -surok_conf = '/etc/surok/conf/surok.json' # Command line arguments parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', help='surok.json path') args = parser.parse_args() -if args.config: - surok_conf = args.config -# Read config file -f = open(surok_conf, 'r') -conf = json.loads(f.read()) -f.close() - - -# Get app configurations -# Return list of patches to app discovery configuration -def get_configs(): - confs = [f for f in listdir(conf['confd']) if isfile( - join(conf['confd'], f))] - return sorted(confs) - - -# Get Surok App configuration -# Read app conf from file and return dict -def load_app_conf(app): - # Load OS environment to app_conf - f = open(conf['confd'] + '/' + app) - c = json.loads(f.read()) - f.close() - - c['env'] = os.environ - - return c - -logger.set_level(conf.get('loglevel','info')) +# Load base configurations +config=Config(args.config if args.config else '/etc/surok/conf/surok.json') # Main loop ########### @@ -56,29 +28,21 @@ logger.set_level(conf.get('loglevel','info')) discovery=Discovery() while 1: - confs = get_configs() - # Update config from discovery object - discovery.set_config(conf) - - # Update discovery data discovery.update_data() + for app in config.apps: - for app in confs: - app_conf = load_app_conf(app) - - # Resolve services - app_hosts = discovery.resolve(app_conf) + app_hosts = discovery.resolve(app) # Populate my dictionary my = {"services": app_hosts, - "conf_name": app_conf['conf_name']} + "conf_name": app['conf_name']} logger.debug('my=',my) # Generate config from template - service_conf = gen(my, app_conf['template']) + service_conf = gen(my, app['template']) - reload_conf(service_conf, app_conf, conf, app_hosts) + reload_conf(service_conf, app, config, app_hosts) - sleep(conf['wait_time']) + sleep(config['wait_time']) diff --git a/surok.spec b/surok.spec index 82289d1..33c1a53 100644 --- a/surok.spec +++ b/surok.spec @@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates} install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf -install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/requriments.txt %{buildroot}/opt/surok @@ -59,7 +61,11 @@ cd /opt/surok && pip3 install -r requriments.txt /opt/surok/surok/templates.py /opt/surok/surok/templates.pyc /opt/surok/surok/templates.pyo -/etc/surok/conf.d/selfcheck.json +/opt/surok/surok/config.py +/opt/surok/surok/config.pyc +/opt/surok/surok/config.pyo +/etc/surok/conf.d/self_check.json +/etc/surok/conf.d/marathon_check.json /etc/surok/templates/selfcheck.jj2 /opt/surok/requriments.txt %defattr(-,root,root,-) diff --git a/surok/config.py b/surok/config.py new file mode 100644 index 0000000..4d58742 --- /dev/null +++ b/surok/config.py @@ -0,0 +1,410 @@ +# Public names +__all__ = ['Config', 'AppConfig'] + +import hashlib +import importlib +import json +import os +from .logger import * + +# Logger link + +# Config singleton link +_config_singleton = None + +''' +Test values +================================================== +key - key +value - value of key +type_value - type of value +type_par - additional parameters for test + +''' + + + +''' +Public Config object +================================================== +.set_config(conf_data) - set config data + Use: conf_data(str type) - path of json config file + conf_data(dict type) - dict with config +.set(key,value) - set config key +.get(key) - get config key +.update_apps() - update apps config data +.apps - Apps object. List of AppConfig oblects +''' + +class _ConfigTemplate(dict): + _conf={} + def _init_conf(self,params): + conf={} + for k in params.keys(): + if params[k].get('params'): + conf[k]=self._init_conf(params[k].get('params')) + else: + if params[k].get('value') is not None: + conf[k]=params[k].get('value') + return conf + + def __init__(self, *conf_data): + self._logger = Logger() + if not self._conf: + self._conf = self._init_conf(self._params) + for c in conf_data: + self.set_config(c) + + def _set_conf_params(self,oldconf,testconf,params): + conf = oldconf if oldconf else {} + for key in testconf.keys(): + resvalue=None + param=params.get(key) + oldvalue=conf.get(key) + testvalue=testconf.get(key) + if param is None: + self._logger.error('Parameter "', key, '" value "', testvalue, '" type is "', type(testvalue).__name__, '" not found') + else: + type_param=param.get('type') + resvalue=[] + if type(testvalue).__name__ != 'list': + testvalue=[testvalue] + for testitem in testvalue: + if self._test_value(key, testitem, param): + if 'dict' in type_param: + if param.get('params'): + res=self._set_conf_params(oldvalue,testitem,param.get('params')) + if res is not None: + resvalue.append(res) + else: + resvalue.append(testitem) + if 'list' not in type_param: + resvalue=list([None]+resvalue).pop() + if resvalue is not None and 'do' in type_param: + if not self._do_type_set(key, resvalue, param): + self._logger.warning('Parameter "', key, '" current "', resvalue, '" type is "', type(resvalue).__name__, '" testing failed') + resvalue = None + if resvalue is not None: + conf[key]=resvalue + return conf + + def _test_value(self, key, value, param): + type_param=param.get('type') + type_value=[x for x in type_param if x in ['str', 'int', 'bool', 'dict']] + if type_value: + if type(value).__name__ not in type_value: + self._logger.error('Parameter "', key, '" must be ', type_value,' types, current "', value, '" (',type(value).__name__,')') + return False + if 'value' in type_param: + if value not in param.get('values',[]): + self._logger.error('Value "', value, '" of key "', key, '" unknown') + return False + if 'dir' in type_param: + if not os.path.isdir(value): + self._logger.error('Path "{}" not present'.format(value)) + return False + elif 'file' in type_param: + if not os.path.isfile(value): + self._logger.error('File "{}" not present'.format(value)) + return False + return True + else: + self._logger.error('Type for testing "{}" unknown'.format(type_value)) + return False + + def set_config(self, conf_data): + conf = {} + if type(conf_data).__name__ == 'str': + try: + self._logger.debug('Open file ', conf_data) + f = open(conf_data, 'r') + json_data=f.read() + f.close() + conf = json.loads(json_data) + except OSError as err: + self._logger.error("OS error: {0}".format(err)) + pass + except ValueError as err: + self._logger.error('JSON format error: {0}'.format(err)) + pass + except: + self._logger.error('Load main config file failed') + pass + elif type(conf).__name__ == 'dict': + conf = conf_data + else: + return False + self._conf=self._set_conf_params(self._conf,conf,self._params) + self._logger.debug('Conf=', self._conf) + + def keys(self): + return self._conf.keys() + + def dump(self): + return json.dumps(self._conf, sort_keys=True, indent=2) + + def _do_type_set(self, key, value, params): + self._logger.error('_do_type_set handler is not defined') + return False + + def hash(self): + return hashlib.sha1(json.dumps(self._conf, sort_keys=True).encode()).hexdigest() + + def set(self, key, value): + self._conf[key]=value + + def __setitem__(self, key, value): + self.set(key, value) + + def get(self, key, default=None): + return self._conf.get(key, default) + + def __getitem__(self, key): + return self.get(key) + + def __contains__(self, item): + return bool(item in self._conf) + + def __len__(self): + return self._conf.__len__() + + def __str__(self): + return self._conf.__str__() + + def __repr__(self): + return self._conf.__repr__() + + +class Config(_ConfigTemplate): + _params = { + 'marathon': { + 'params': { + 'force': { + 'value': True, + 'type': ['bool'] + }, + 'host': { + 'value': 'http://marathon.mesos:8080', + 'type': ['str'] + }, + 'enabled': { + 'value': False, + 'type': ['bool'] + }, + 'restart': { + 'value': False, + 'type': ['bool'] + } + }, + 'type':['dict'] + }, + 'mesos': { + 'params': { + "domain": { + 'value': "marathon.mesos", + 'type': ['str'] + }, + "enabled": { + 'value': False, + 'type': ['bool'] + } + }, + 'type':['dict'] + }, + 'memcached': { + 'params': { + 'enabled': { + 'value': False, + 'type': ['bool'] + }, + 'discovery': { + 'params': { + 'enabled': { + 'value': False, + 'type': ['bool'] + }, + 'service': { + 'value': 'memcached', + 'type': ['str'] + }, + 'group': { + 'value': 'system', + 'type': ['str'] + } + }, + 'type':['dict'] + }, + 'hosts': { + 'value': ['localhost:11211'], + 'type': ['list','str'] + } + }, + 'type':['dict'] + }, + 'version': { + 'value': '0.7', + 'type': ['str','value'], + 'values': ['0.7', '0.8'] + }, + 'confd': { + 'value': '/etc/surok/conf.d', + 'type': ['str','dir'] + }, + 'wait_time': { + 'value': 20, + 'type': ['int'] + }, + 'lock_dir': { + 'value': '/var/tmp', + 'type': ['str','dir'] + }, + 'default_discovery': { + 'value': 'mesos_dns', + 'type': ['str', 'value'], + 'values': ['mesos_dns', 'marathon_api'] + }, + 'loglevel': { + 'value': 'info', + 'type': ['str','do'], + 'do':'set_loglevel' + } + } + + def __new__(cls, *args): + global _config_singleton + if _config_singleton is None: + _config_singleton = super(Config, cls).__new__(cls) + return _config_singleton + + def __init__(self, *conf_data): + super().__init__( *conf_data) + self.apps = _Apps() + + def set_config(self, conf_data): + super().set_config(conf_data) + if self.get('version') == '0.7': + domain = self.get('domain') + if domain is not None and self.get('mesos') is None: + self.set('mesos', {'domain': domain, 'enabled': True}) + + def _do_type_set(self, key, value, param): + if param.get('do') == 'set_loglevel': + if self._logger.set_level(value): + return True + return False + + def update_apps(self): + self.apps.reset() + for app in sorted([os.path.join(self.get('confd'), f) for f in os.listdir(self.get('confd')) if os.path.isfile(os.path.join(self.get('confd'), f))]): + self.apps.set(AppConfig(app)) + +''' +Private Apps object +================================================== +.get(app) - get _AppConfig object +.set(app) - set _AppConfig object +''' + + +class _Apps: + _apps = {} + _items = [] + + def get(self, key): + return self._apps.get(key) + + def set(self, app): + self._apps[app.get('conf_name')] = app + + def reset(self): + keys=[]+list(self.keys()) + for k in keys: + del self._apps[k] + + def __getitem__(self, key): + return self.get(key) + + def __contains__(self, item): + return bool(item in self._apps) + + def __iter__(self): + self._items = sorted(self.keys()) + return self + + def __next__(self): + if self._items: + return self.get(self._items.pop(0)) + raise StopIteration + + def keys(self): + return self._apps.keys() + +''' +Public AppConfig object +================================================== +.set_config(conf_data) - set config data + Use: conf_data(str type) - path of json config file + conf_data(dict type) - dict with config +.set(key,value) - set config key +.get(key) - get config key +''' + + +class AppConfig(_ConfigTemplate): + _params = { + 'conf_name': { + 'type': ['str'] + }, + 'services': { + 'params': { + 'name': { + 'type': ['str'] + }, + 'ports': { + 'type': ['list','str'] + }, + 'discovery': { + 'type': ['str'] + }, + 'group': { + 'type': ['str'] + } + }, + 'type':['list','dict'] + }, + 'template': { + 'type': ['str','file'] + }, + 'dest': { + 'type': ['str'] + }, + 'reload_cmd': { + 'type': ['str'] + }, + 'discovery': { + 'type': ['str'] + }, + 'group': { + 'type': ['str'] + } + } + + def __init__(self, *conf_data): + self._config = Config() + super().__init__(*conf_data) + + def set_config(self, conf_data): + super().set_config(conf_data) + self._conf.setdefault('discovery', self._config.get('default_discovery')) + self._conf.setdefault('group', self._get_default_group()) + if type(conf_data).__name__ == 'str': + self._conf.setdefault('conf_name', os.path.basename(conf_data)) + + def _get_default_group(self): + env=self._config.get('env',dict(os.environ)) + # Check environment variable + if env.get('SUROK_DISCOVERY_GROUP'): + return env['SUROK_DISCOVERY_GROUP'] + # Check marathon environment variable + elif env.get('MARATHON_APP_ID'): + return ".".join(env['MARATHON_APP_ID'].split('/')[-2:0:-1]) diff --git a/surok/discovery.py b/surok/discovery.py index 429e51b..8018b7f 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -1,282 +1,217 @@ +#Public names +__all__=['Discovery','DiscoveryMesos','DiscoveryMarathon'] import dns.resolver import dns.query -from dns.exception import DNSException -from .logger import Logger +import os import sys import requests +from dns.exception import DNSException +from .config import * +from .logger import * -# Default config for Discovery class -_config={ - 'default_discovery':'mesos_dns' # Default discovery system -} - -# Discoveries objects -_discoveries={} - -#Logger -logger=Logger() +# Discovery object +_discovery_singleton=None class DiscoveryTemplate: - # Default config values for discovery template - _config={} - _defconfig={'enabled':False} - - def __init__(self,conf): - for key in self._defconfig.keys(): - if key not in self._config.keys(): - self._config[key]=self._defconfig[key] - self.set_config(conf) - - def set_config(self,conf): - pass + def __init__(self): + self._config=Config() + self._logger=Logger() def enabled(self): - return self._config['enabled'] + return self._config[self._config_section].get('enabled',False) def update_data(self): pass - def get_group(self,service, app): - # Check group in app conf - if 'group' in service: - return service['group'] + # Do DNS queries + # Return array: + # ["10.10.10.1", "10.10.10.2"] + def do_query_a(self,fqdn): + servers = [] + try: + resolver = dns.resolver.Resolver() + for a_rdata in resolver.query(fqdn, 'A'): + servers.append(a_rdata.address) + except DNSException as e: + self._logger.error("Could not resolve ",fqdn) + return servers - # Check environment variable - elif app['env'].get('SUROK_DISCOVERY_GROUP'): - return app['env']['SUROK_DISCOVERY_GROUP'] - - # Check marathon environment variable - elif app['env'].get('MARATHON_APP_ID'): - return ".".join(app['env']['MARATHON_APP_ID'].split('/')[-2:0:-1]) - - else: - logger.error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID') - logger.error('Not in Mesos launch?') - sys.exit(2) + # Do DNS queries + # Return array: + # [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}] + def do_query_srv(self,fqdn): + servers = [] + try: + resolver = dns.resolver.Resolver() + resolver.lifetime = 1 + resolver.timeout = 1 + query = resolver.query(fqdn, 'SRV') + for rdata in query: + info = str(rdata).split() + servers.append({'name': info[3][:-1], 'port': info[2]}) + except DNSException as e: + self._logger.warning("Could not resolve ",fqdn) + return servers class Discovery: - def __init__(self,*conf): - for __conf in conf: - self.set_config(__conf) + _discoveries={} + def __new__(cls): + global _discovery_singleton + if _discovery_singleton is None: + _discovery_singleton=super(Discovery, cls).__new__(cls) + return _discovery_singleton - def set_config(self,conf): - global _discoveries - #Get discoveries objects - if not _discoveries.get('mesos_dns'): - _discoveries['mesos_dns']=DiscoveryMesos(conf) - else: - _discoveries['mesos_dns'].set_config(conf) + def __init__(self): + self._config=Config() + self._logger=Logger() + if not self._discoveries.get('mesos_dns'): + self._discoveries['mesos_dns']=DiscoveryMesos() + if not self._discoveries.get('marathon_api'): + self._discoveries['marathon_api']=DiscoveryMarathon() - if not _discoveries.get('marathon_api'): - _discoveries['marathon_api']=DiscoveryMarathon(conf) - else: - _discoveries['marathon_api'].set_config(conf) - - if not _discoveries.get('consul_dns'): - _discoveries['consul_dns']=DiscoveryConsul(conf) - else: - _discoveries['consul_dns'].set_config(conf) - - global _config - if conf.get('default_discovery'): - discovery=conf.get('default_discovery') - if discovery in list(_discoveries.keys()): - _config['default_discovery']=discovery - else: - logger.error('Default discovery "'+discovery+'" is not present') - logger.debug('Conf=',conf) + def keys(self): + return self._discoveries.keys() def resolve(self,app): - __discovery=_config.get('default_discovery') - if app.get('discovery'): - discovery=app.get('discovery') - if discovery in list(_discoveries.keys()): - __discovery=discovery - else: - logger.warning('Discovery "'+discovery+'" is not present') - logger.debug('App=',app) - return {} - if _discoveries[__discovery].enabled(): - return _discoveries[__discovery].resolve(app) + discovery=app.get('discovery',self._config.get('default_discovery')) + if discovery not in self.keys(): + self._logger.warning('Discovery "',discovery,'" is not present') + return {} + if self._discoveries[discovery].enabled(): + return self.compatible(self._discoveries[discovery].resolve(app)) else: - logger.error('Discovery "'+__discovery+'" is disabled') + self._logger.error('Discovery "',discovery,'" is disabled') return {} def update_data(self): - global _discoveries - for d in list(_discoveries.keys()): - if _discoveries[d].enabled(): - _discoveries[d].update_data() + self._config.update_apps() + for d in self.keys(): + if self._discoveries[d].enabled(): + self._discoveries[d].update_data() + + def compatible(self,hosts): + compatible_hosts={} + if self._config.get('version') == '0.7': + for service in hosts.keys(): + for host in hosts[service]: + ports=host.get('tcp',[]) + if type(ports).__name__ == 'list': + compatible_hosts[service]=[] + for port in ports: + compatible_hosts[service].append({'name':host['name'], + 'ip':host['ip'], + 'port':str(port)}) + else: + compatible_hosts[service]={} + for port in ports.keys(): + compatible_host=compatible_hosts[service].setdefault(port,[]) + compatible_host.append({'name':host['name'], + 'ip':host['ip'], + 'port':ports[port]}) + + return compatible_hosts + return hosts class DiscoveryMesos(DiscoveryTemplate): - _config={ - 'domain':'marathon.mesos' # Default domain - } - - def set_config(self,conf): - # For old version config - if conf.get('domain'): - self._config['domain']=conf.get('domain') - self._config['enabled']=True - # For current version config - if conf.get('mesos'): - _conf=conf['mesos'] - for p in ['domain','enabled']: - if _conf.get(p): - self._config[p]=_conf.get(p) - + _config_section='mesos' def resolve(self,app): hosts = {} - services = app['services'] - domain = self._config['domain'] + services = app.get('services') + domain = self._config[self._config_section].get('domain') for service in services: - group = self.get_group(service, app) + group = service.get('group',app.get('group')) + if group is None: + self._logger.error('Group for service "{}" of config "{}" not found'.format(service['name'],app.get('conf_name'))) + continue ports = service.get('ports') name = service['name'] hosts[name] = {} serv = hosts[name] - if ports is not None: - hosts[name] = {} - serv = hosts[name] - for prot in ['tcp','udp']: + self._logger.debug('group=',group,' ports=',ports,' name=',name,' serv=',serv) + for prot in ['tcp','udp']: + if ports is not None: for port_name in ports: - for d in do_query('_'+port_name+'._'+name+'.'+group+'._'+prot+'.'+domain): + for d in self.do_query_srv('_'+port_name+'._'+name+'.'+group+'._'+prot+'.'+domain): hostname=d['name'] - if serv.get(hostname) is None: - serv[hostname]={"name":hostname,"ip":d['ip']} - if serv[hostname].get(prot) is None: - serv[hostname][prot]={} - serv[hostname][prot][port_name]=d['port'] - hosts[name]=list(hosts[name].values()) - else: - hosts[name]=do_query('_'+name+'.'+group+'._tcp.'+domain) - + serv.setdefault(hostname,{'name':hostname, + 'ip':self.do_query_a(hostname)}) + serv[hostname].setdefault(prot,{}) + serv[hostname][prot][port_name]=d['port'] + else: + for d in self.do_query_srv('_'+name+'.'+group+'._'+prot+'.'+domain): + hostname=d['name'] + if serv.get(hostname) is None: + serv[hostname]={'name':hostname, + 'ip':self.do_query_a(hostname)} + if serv[hostname].get(prot) is None: + serv[hostname][prot]=[] + serv[hostname][prot].extend([d['port']]) + hosts[name]=list(serv.values()) return hosts class DiscoveryMarathon(DiscoveryTemplate): - _config={ - 'host':'http://marathon.mesos:8080', - 'force':True - } - __tasks = [] - __ports = {} - def set_config(self,conf): - # For current version config - if conf.get('marathon'): - _conf=conf['marathon'] - for p in ['host','enabled','force']: - if _conf.get(p): - self._config[p]=_conf.get(p) + _config_section='marathon' + _tasks = [] + _ports = {} def update_data(self): + hostname=self._config[self._config_section].get('host') try: - apps = requests.get(self._config['host']+'/v2/apps').json()['apps'] ports = {} - for app in apps: + for app in requests.get(hostname+'/v2/apps').json()['apps']: ports[app['id']] = {} - if app.get('container') is not None and app['container']['type'] == 'DOCKER': + if app.get('container') is not None and app['container'].get('type') == 'DOCKER': ports[app['id']] = app['container']['docker'].get('portMappings',[]) - self.__ports=ports + self._ports=ports except: - logger.warning('Apps ('+self._config['host']+'/v2/apps) request from Marathon API is failed') + self._logger.warning('Apps (',hostname,'/v2/apps) request from Marathon API is failed') pass try: - self.__tasks = requests.get(self._config['host']+'/v2/tasks').json()['tasks'] + self._tasks = requests.get(hostname + '/v2/tasks').json()['tasks'] except: - logger.warning('Tasks ('+self._config['host']+'/v2/tasks) request from Marathon API is failed') + self._logger.warning('Tasks (',hostname,'/v2/tasks) request from Marathon API is failed') pass + def _test_mask(self, mask, value): + return (mask.endswith('*') and value.startswith(mask[:-1])) or mask == value + def resolve(self, app): hosts={} - serv_conf = app['services'] - if not serv_conf: - serv_conf = [{'name':'*','ports':['*']}] - for serv in serv_conf: - # Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format - group = '/'.join(['']+self.get_group(serv, app).split('.')[::-1]+['']) - mask = group+serv['name'] - for task in self.__tasks: - if (mask.endswith('*') and task['appId'].startswith(mask[:-1])) or task['appId'] == mask: - name='.'.join(task['appId'][len(group):].split('/')[::-1]) - if 'ports' in serv: - hosts[name]={} - for port in self.__ports[task['appId']]: - for pp in serv['ports']: - if (pp.endswith('*') and port['name'].startswith(pp[:-1])) or port['name'] == pp: - if hosts[name].get(task['host']) is None: - hosts[name][task['host']]={'name':task['host'], - 'ip':do_query_a(task['host'])} - if hosts[name][task['host']].get(port['protocol']) is None: - hosts[name][task['host']][port['protocol']]={} - hosts[name][task['host']][port['protocol']][port['name']]=task['ports'][task['servicePorts'].index(port['servicePort'])] - hosts[name]=list(hosts[name].values()) - else: - hosts[name]=[] - for port in self.__ports[task['appId']]: - hosts[name].append({'name':task['host'], - 'port':task['ports'][task['servicePorts'].index(port['servicePort'])], - 'ip':do_query_a(task['host'])}) - - return hosts - - -class DiscoveryConsul(DiscoveryTemplate): - _config={ - 'enabled':False, - 'domain':None - } - def set_config(self,conf): - # For current version config - if conf.get('consul'): - _conf=conf['consul'] - for p in ['domain','enabled']: - if _conf.get(p): - self._config[p]=_conf.get(p) - - def resolve(self,app): - hosts = {} - services = app['services'] - domain = self._config['domain'] + services = app.get('services') + if not services: + services = [{'name':'*','ports':['*']}] for service in services: - name = service['name'] - hosts[name]=do_query('_'+name+'._tcp.'+domain) + # Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format + group=service.get('group',app.get('group')) + if group is None: + self._logger.error('Group for service "{}" of config "{}" not found'.format(service['name'],app.get('conf_name'))) + continue + group = '/'+'/'.join(group.split('.')[::-1])+'/' + service_mask = group + service['name'] + for task in self._tasks: + if self._test_mask(service_mask,task['appId']): + name='.'.join(task['appId'][len(group):].split('/')[::-1]) + hosts[name]={} + serv = hosts[name] + hostname=task['host'] + for task_port in self._ports[task['appId']]: + prot=task_port['protocol'] + port_name=task_port['name'] + port=task['ports'][task['servicePorts'].index(task_port['servicePort'])] + if 'ports' in service: + for port_mask in service['ports']: + if self._test_mask(port_mask,port_name): + serv.setdefault(hostname,{'name':hostname, + 'ip':self.do_query_a(hostname)}) + serv[hostname].setdefault(prot,{}) + serv[hostname][prot][port_name]=port + else: + serv.setdefault(hostname,{'name':hostname, + 'ip':self.do_query_a(hostname)}) + serv[hostname].setdefault(prot,[]) + serv[hostname][prot].extend([port]) + hosts[name]=list(serv.values()) return hosts - - -# Do DNS queries -# Return array: -# ["10.10.10.1", "10.10.10.2"] -def do_query_a(fqdn): - servers = [] - try: - resolver = dns.resolver.Resolver() - for a_rdata in resolver.query(fqdn, 'A'): - servers.append(a_rdata.address) - except DNSException as e: - logger.error("Could not resolve "+fqdn) - - return servers - - -# Do DNS queries -# Return array: -# [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}] -def do_query(fqdn): - servers = [] - try: - resolver = dns.resolver.Resolver() - resolver.lifetime = 1 - resolver.timeout = 1 - query = resolver.query(fqdn, 'SRV') - for rdata in query: - info = str(rdata).split() - name = info[3][:-1] - port = info[2] - servers.append({'name': name, 'port': port, 'ip': do_query_a(name)}) - except DNSException as e: - logger.error("Could not resolve " + fqdn) - - return servers diff --git a/surok/logger.py b/surok/logger.py index 7cee349..eec8efb 100644 --- a/surok/logger.py +++ b/surok/logger.py @@ -1,51 +1,86 @@ +# Public names +__all__ = ['Logger'] + import sys import json -from time import time -_loglevel='info' -msg_level={'debug':'DEBUG', - 'info':'INFO', - 'warning':'WARNING', - 'error':'ERROR'} +import time + +# Logger singleton link +_logger_singleton = None + +''' +Public Logger oblect +================================================== +.set_level(level) - set level messages + level - values 'debug', 'info', 'warning', 'error' + * error - write error message + * warning - write warning and error message + * info - write info, warning and error message + * debug - write all message +.get_level() - get level messages +.error(str) - write error message +.warning(str) - write warning message +.info(str) - write info message +.debug(str)- write error message +''' + class Logger: - def __init__(self,*args): + _loglevel = 'info' + _msg_level = { + 'debug': 'DEBUG', + 'info': 'INFO', + 'warning': 'WARNING', + 'error': 'ERROR' + } + + def __new__(cls, *args): + global _logger_singleton + if _logger_singleton is None: + _logger_singleton = super(Logger, cls).__new__(cls) + return _logger_singleton + + def __init__(self, *args): if args: self.set_level(args[0]) - def set_level(self,level): - if level in ['debug','info','warning','error']: - global _loglevel - _loglevel=level + def set_level(self, level): + if level in ['debug', 'info', 'warning', 'error']: + self._loglevel = level + return True + else: + self.warning('Log level "', level, '" not valid') + return False def get_level(self): - return _loglevel + return self._loglevel - def __make_message(self,message): - r=[] - l=self.get_level() + def _make_message(self, level, message): + r = [] for m in message: - if type(m).__name__=='str': + if type(m).__name__ == 'str': r.append(m) else: - r.append(json.dumps(m,sort_keys=True,indent=2)) - return '[' + str(time()) + '] ' + msg_level[l] + ': ' + ''.join(r) + "\n" + r.append(json.dumps(m, sort_keys=True, indent=2)) + return '[' + str(time.time()) + '] ' + self._msg_level[level] + ': ' + ''.join(r) + "\n" - def debug(self,*message): + def debug(self, *message): if self.get_level() in ['debug']: - sys.stderr.write(self.__make_message(message)) + self._log2err(self._make_message('debug',message)) - def info(self,*message): - if self.get_level() in ['debug','info']: - sys.stdout.write(self.__make_message(message)) + def info(self, *message): + if self.get_level() in ['debug', 'info']: + self._log2out(self._make_message('info',message)) - def warning(self,*message): - if self.get_level() in ['debug','info','warning']: - sys.stderr.write(self.__make_message(message)) + def warning(self, *message): + if self.get_level() in ['debug', 'info', 'warning']: + self._log2out(self._make_message('warning',message)) - def error(self,*message): - sys.stderr.write(self.__make_message(message)) + def error(self, *message): + self._log2err(self._make_message('error',message)) - def testing(self,level,message): - self.set_level(level) - return self.__make_message(message) + def _log2err(self,out): + sys.stderr.write(out) + def _log2out(self,out): + sys.stdout.write(out) diff --git a/surok/system.py b/surok/system.py index 89b1466..81717b5 100644 --- a/surok/system.py +++ b/surok/system.py @@ -3,6 +3,7 @@ import sys import requests from .discovery import Discovery from .logger import Logger + logger=Logger() # Get old configuration @@ -91,7 +92,8 @@ def discovery_memcached(conf): # !!! NEED REFACTORING !!! def reload_conf(service_conf, app_conf, conf, app_hosts): # Check marathon enabled in configuration - if conf['marathon'].get('restart',False): + if (conf.get('version','0.7')=='0.8' and conf['marathon'].get('restart',False)) or ( + conf.get('version','0.7')=='0.7' and conf['marathon'].get('enable',False)): if get_old(app_conf['conf_name'], service_conf) != 1: restart_self_in_marathon(conf['marathon']) diff --git a/surok_fedora.spec b/surok_fedora.spec index 57f839f..ee5373f 100644 --- a/surok_fedora.spec +++ b/surok_fedora.spec @@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates} install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf -install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates @@ -43,7 +45,9 @@ rm -rf $RPM_BUILD_ROOT /opt/surok/surok/discovery.py /opt/surok/surok/system.py /opt/surok/surok/templates.py -/etc/surok/conf.d/selfcheck.json +/opt/surok/surok/config.py +/etc/surok/conf.d/self_check.json +/etc/surok/conf.d/marathon_check.json /etc/surok/templates/selfcheck.jj2 %defattr(-,root,root,-) %doc