diff --git a/AUTHORS b/AUTHORS index a720639..354c3b8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,2 +1,3 @@ Denis Zheleztsov Denis Ryabyy +Evgeniy Vasilev diff --git a/README.ru.md b/README.ru.md index 022ddc4..66adf3f 100644 --- a/README.ru.md +++ b/README.ru.md @@ -1,5 +1,5 @@ # Surok -[![Build Status](https://travis-ci.org/Difrex/surok.svg?branch=master)](https://travis-ci.org/Difrex/surok) +[![Build Status](https://travis-ci.org/Surkoveds/surok.svg?branch=master)](https://travis-ci.org/Surkoveds/surok) Обнаружение сервисов для Apache Mesos. @@ -17,10 +17,27 @@ cd build deb-пакет будет лежать в build/out Сборка базового docker-образа surok +Ubuntu Xenial ``` cd build ./build.sh surok_image ``` +Alpine image +``` +cd build +./build.sh alpine +``` +CentOS image +``` +cd build +./build.sh centos +``` + +ENTRYPOINT : ```cd /opt/surok && pytho3 surok.py -c /etc/surok/conf/surok.json``` + +## Документация + +[Wiki](https://github.com/Surkoveds/surok/wiki) ## Известные проблемы diff --git a/build/runtests.sh b/build/runtests.sh index 37cbbe7..f54968f 100755 --- a/build/runtests.sh +++ b/build/runtests.sh @@ -5,7 +5,7 @@ set -e . functions.sh function run_tests() { - docker run -ti -v $(pwd)/tests.py:/opt/surok/tests.py \ + docker run --rm -ti -v $(pwd)/tests.py:/opt/surok/tests.py \ -v $(pwd)/tests_entrypoint.sh:/tests_entrypoint.sh \ --entrypoint /tests_entrypoint.sh \ surok_base:latest diff --git a/build/tests.py b/build/tests.py old mode 100644 new mode 100755 index ece5c7b..b8d1dd0 --- a/build/tests.py +++ b/build/tests.py @@ -1,71 +1,372 @@ +#!/usr/bin/python3 import unittest import json import os import re +import sys +import surok.config +import surok.logger +import surok.discovery +import hashlib +from surok.config import Config + +class Logger(surok.logger.Logger): + _out='' + _err='' + def _log2err(self,out): + self._err+=out + + def _log2out(self,out): + self._out+=out + + def geterr(self): + return self._err + + def getout(self): + return self._out + + def reset(self): + self._err='' + self._out='' + +class DiscoveryTestingTemplate: + _testing={} + _testing_fqdn_a={ + "test.zzz0.test":['10.0.0.1','10.1.0.1'], + "test.zzz1.test":['10.0.1.1','10.1.1.1'], + "test.zzz2.test":['10.0.2.1','10.1.2.1'], + "test.zzz3.test":['10.0.3.1','10.1.3.1'] + } + _testing_fqdn_srv={} + + def __init__(self): + super().__init__() + self._orig_logger=surok.logger.Logger() + + def do_query_a(self,fqdn): + res=self._testing_fqdn_a.get(fqdn,[]) + if res: + return res + else: + self._orig_logger.error('Testing FQDN '+fqdn+' not found in test A records') + sys.exit(2) + + def do_query_srv(self,fqdn): + res=self._testing_fqdn_srv.get(fqdn,[]) + if res or fqdn.startswith('_tname_e.') or fqdn.find('._udp.'): + return res + else: + self._orig_logger.error('Testing FQDN '+fqdn+' not found in test SRV records') + sys.exit(2) + + def update_data(self): + class_name=self.__class__.__name__ + tgen={ + "name": ["zzz0","zzy0","zzy1","zzz1"], + "host": ["test.zzz0.test","test.zzz1.test","test.zzz2.test","test.zzz3.test"], + "serv": ["tname_aa","tname_ab","tname_ba","tname_bb"], + "ports": [12341,12342,12343,12344], + "servicePorts": [21221,21222,21223,21224] + } + if self._testing.get(class_name,True): + if class_name == 'DiscoveryMarathon': + _tasks=[] + _ports={} + for id in (0,1,2,3): + ports=[]+tgen['ports'] + servicePorts=[]+tgen['servicePorts'] + appId='/'.join(str(tgen['name'][id]+'.xxx.yyy.').split('.')[::-1]) + _ports[appId]=[] + for pid in (0,1,2,3): + ports[pid]+=pid*10 + servicePorts[pid]+=pid*100 + for prot in ['tcp','udp']: + if pid<2 or prot == 'tcp': + _ports[appId].append({'containerPort': 0, + 'hostPort': 0, + 'labels': {}, + 'name': tgen['serv'][pid], + 'protocol': prot, + 'servicePort': servicePorts[pid]}) + + _tasks.append({'appId':appId, + 'host':tgen['host'][id], + 'ports':ports, + 'servicePorts':servicePorts}) + #_tname_a._zzy0.yyy.xxx._tcp.marathon.mesos + self._tasks=_tasks + self._ports=_ports + elif class_name == 'DiscoveryMesos': + for id in (0,1,2,3): + ports=[]+tgen['ports'] + for pid in (0,1,2,3): + ports[pid]+=pid*10 + for prot in ['tcp','udp']: + if pid<2 or prot == 'tcp': + for fqdn in ['_'+tgen['serv'][pid]+'._'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain'), + '_'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain')]: + if not self._testing_fqdn_srv.get(fqdn): + self._testing_fqdn_srv[fqdn]=[] + self._testing_fqdn_srv[fqdn].append({'name':tgen['host'][id],'port':ports[pid]}) + + self._testing[class_name]=False + +class DiscoveryMesos(DiscoveryTestingTemplate,surok.discovery.DiscoveryMesos): + pass + +class DiscoveryMarathon(DiscoveryTestingTemplate,surok.discovery.DiscoveryMarathon): + pass + +class Discovery(surok.discovery.Discovery): + _discoveries={} + def __init__(self): + self._config=Config() + self._logger=Logger() + if not self._discoveries.get('mesos_dns'): + self._discoveries['mesos_dns']=DiscoveryMesos() + if not self._discoveries.get('marathon_api'): + self._discoveries['marathon_api']=DiscoveryMarathon() + +class Test01_Logger(unittest.TestCase): + def test_01_logger_default_level(self): + logger = Logger() + self.assertEqual(logger.get_level(), 'info') + + def test_02_logger_output_levels(self): + message='log message' + tests={ + 'debug':{ + 'assertIn':['ERROR: {}','WARNING: {}','INFO: {}','DEBUG: {}'], + 'assertNotIn':[] + }, + 'info':{ + 'assertIn':['ERROR: {}','WARNING: {}','INFO: {}'], + 'assertNotIn':['DEBUG: {}'] + }, + 'warning':{ + 'assertIn':['ERROR: {}','WARNING: {}'], + 'assertNotIn':['INFO: {}','DEBUG: {}'] + }, + 'error':{ + 'assertIn':['ERROR: {}'], + 'assertNotIn':['WARNING: {}','INFO: {}','DEBUG: {}'] + } + } + logger = Logger() + for value01 in tests.keys(): + logger.reset() + logger.set_level(value01) + logger.error(message) + logger.warning(message) + logger.info(message) + logger.debug(message) + resmessage=logger.geterr()+logger.getout() + for test_name in tests[value01].keys(): + for test_value in tests[value01][test_name]: + with self.subTest(msg='Testing Logger for ...', loglevel=value01): + test_message=test_value.format(message) + eval('self.{}(test_message,resmessage)'.format(test_name)) + +class Test02_LoadConfig(unittest.TestCase): + + def test_01_default_values(self): + config=Config() + with self.subTest(msg='Testing default values for Config.', dump=config.dump()): + self.assertEqual(config.get('confd'), '/etc/surok/conf.d') + self.assertEqual(config.get('default_discovery'), 'mesos_dns') + self.assertEqual(config.get('lock_dir'), '/var/tmp') + self.assertEqual(config.get('loglevel'), 'info') + self.assertEqual(dict(config.get('marathon',{})).get('enabled'), False) + self.assertEqual(dict(config.get('mesos',{})).get('enabled'), False) + self.assertEqual(dict(config.get('memcached',{})).get('enabled'), False) + self.assertEqual(config.get('version'), '0.7') + self.assertEqual(config.get('wait_time'), 20) + + def test_02_main_conf(self): + config=Config('/etc/surok/conf/surok.json') + with self.subTest(msg='Testing load config for Config.', dump=config.dump()): + self.assertEqual(config.hash(), '545c20b322a6ba5fef9c7d2416d80178f26a924b') + + def test_03_apps_conf(self): + tests=[ + { + 'env':{}, + 'self_check.json':'a4e109b9fec696776fd3df091b607e9c1489748c', + 'marathon_check.json':'6be7f26d421d4a0a2e7b089184be0c0e3a50f986' + }, + { + 'env':{'SUROK_DISCOVERY_GROUP':'xxx.yyy'}, + 'self_check.json':'38ab770ff2ba69bf70673288425337ff3c18a807', + 'marathon_check.json':'7b0cb4eab2d8e0f901cc567df28b17279af21baa' + }, + { + 'env':{'MARATHON_APP_ID':'/xxx/yyy/zzz'}, + 'self_check.json':'cbd2a15179649d0e06f98bd64e024481a944d65c', + 'marathon_check.json':'08a382d14285feb1f22b92ba597ecf73d654a2e0' + } + ] + config=Config() + for test in tests: + config.set('env',test['env']) + config.update_apps() + for app in config.apps: + with self.subTest(msg='Testing AppConfig for ...', env=test['env'], conf_name=app.get('conf_name'), dump=app.dump()): + self.assertEqual(test[app.get('conf_name')],app.hash()) + + def test_04_apps_conf(self): + tests={ + 'confd':{ + 'assertEqual': ['/var', '/var/tmp', '/etc/surok/conf.d'], + 'assertNotEqual': [20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True] + }, + 'default_discovery':{ + 'assertEqual':['marathon_api', 'mesos_dns'], + 'assertNotEqual':[20, 'test', None] + }, + 'lock_dir':{ + 'assertEqual':['/var', '/etc/surok/conf.d', '/var/tmp'], + 'assertNotEqual':[20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True] + }, + 'loglevel':{ + 'assertEqual':['error', 'debug', 'info', 'warning'], + 'assertNotEqual':['errrr', 'DEBUG','warn', 'test', 1, None, True] + }, + 'version':{ + 'assertEqual': ['0.7', '0.8'], + 'assertNotEqual': ['0,7', '07', '0.9', 0.7, 0.8, None] + }, + 'wait_time':{ + 'assertEqual': [10, 15, 20], + 'assertNotEqual': ['10', '15', None, True] + } + + } + config=Config() + for name01 in tests.keys(): + oldvalue=config.get(name01) + for test_name in tests[name01].keys(): + for value01 in tests[name01][test_name]: + config.set_config({name01:value01}) + test_value=config.get(name01) + with self.subTest(msg='Testing Config Change for values...', name=name01, value=value01, test_value=test_value): + eval('self.{}(value01, test_value)'.format(test_name)) + config.set(name01,oldvalue) -class TestLoadConfig(unittest.TestCase): +class Test03_Discovery(unittest.TestCase): - def test_main_conf(self): - # Load base configurations - surok_conf = '/etc/surok/conf/surok.json' - # Read config file - f = open(surok_conf, 'r') - conf = json.loads(f.read()) - f.close() - - self.assertIn('confd', conf) - self.assertTrue(os.path.isdir(conf['confd'])) - self.assertIn('domain', conf) - self.assertIn('wait_time', conf) - self.assertIn('lock_dir', conf) - self.assertTrue(os.path.isdir(conf['lock_dir'])) - - -class TestLogger(unittest.TestCase): - - def test_info(self): - from surok.logger import make_message - m = make_message - self.assertIn('INFO', m({'level': 'INFO', 'raw': 'log message'})) - - def test_warning(self): - from surok.logger import make_message - m = make_message - self.assertIn('WARNING', m({'level': 'WARNING', 'raw': 'log message'})) - - def test_error(self): - from surok.logger import make_message - m = make_message - self.assertIn('ERROR', m({'level': 'ERROR', 'raw': 'log message'})) - - def test_info(self): - from surok.logger import make_message - m = make_message - self.assertIn('DEBUG', m({'level': 'DEBUG', 'raw': 'log message'})) - - -class TestMemcachedDiscovery(unittest.TestCase): - - def test_discovery_memcache(self): - from surok.system import discovery_memcached - - # Load base configurations - surok_conf = '/etc/surok/conf/surok.json' - # Read config file - f = open(surok_conf, 'r') - conf = json.loads(f.read()) - f.close() - - self.assertEqual(discovery_memcached(conf), []) - - -class TestGetGroup(unittest.TestCase): - - def test_get_group(self): - from surok.discovery import get_group - self.assertFalse(get_group({}, {'env': os.environ})) - + def test_01_discovery(self): + tests={ + 'T':{ #mesos_enabled + 'T':{ #marathon_enabled + '0.7':{ #version + 'mesos_dns':{ #default_discovery + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', #app['conf_name'] + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + }, + 'marathon_api':{ + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + }, + 'marathon_api':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + } + } + }, + 'F':{ + '0.7':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + } + } + }, + 'F':{ + 'T':{ + '0.7':{ + 'mesos_dns':{ + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + } + } + }, + 'F':{ + '0.7':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + } + } + } + } + config=Config('/etc/surok/conf/surok.json') + config.set('env',{'SUROK_DISCOVERY_GROUP':'xxx.yyy'}) + discovery=Discovery() + for mesos_enabled in tests.keys(): + for marathon_enabled in tests[mesos_enabled].keys(): + for version in tests[mesos_enabled][marathon_enabled].keys(): + for default_discovery in tests[mesos_enabled][marathon_enabled][version].keys(): + config.set_config({'default_discovery':default_discovery, + 'mesos':{'enabled':(mesos_enabled=='T')}, + 'marathon':{'enabled':(marathon_enabled=='T')}, + 'version':version}) + discovery.update_data() + for app in config.apps: + conf_name=app.get('conf_name') + with self.subTest(msg='Testing Discovery for values...', config=config.dump(), conf_name=conf_name): + self.assertEqual(hashlib.sha1(json.dumps(discovery.resolve(app), sort_keys=True).encode()).hexdigest(), + tests[mesos_enabled][marathon_enabled][version][default_discovery][conf_name]) if __name__ == '__main__': unittest.main() diff --git a/conf.d/marathon_check.json b/conf.d/marathon_check.json new file mode 100644 index 0000000..15e784f --- /dev/null +++ b/conf.d/marathon_check.json @@ -0,0 +1,12 @@ +{ + "services": [ + { + "name": "zzy*", + "ports": ["tname_a*"] + } + ], + "template": "templates/selfcheck.jj2", + "dest": "selfcheck", + "reload_cmd": "/bin/echo selfcheck ok", + "discovery": "marathon_api" +} diff --git a/conf.d/self_check.json b/conf.d/self_check.json new file mode 100644 index 0000000..4cf0f65 --- /dev/null +++ b/conf.d/self_check.json @@ -0,0 +1,22 @@ +{ + "services": [ + { + "name": "zzy0", + "ports": ["tname_aa", "tname_ab", "tname_ba", "tname_bb", "tname_d"] + }, + { + "name": "zzy1", + "ports": ["tname_aa", "tname_ab", "tname_ba", "tname_bb"] + }, + { + "name": "zzz0", + "ports": ["tname_aa", "tname_bb"] + }, + { + "name": "zzz1" + } + ], + "template": "templates/selfcheck.jj2", + "dest": "selfcheck", + "reload_cmd": "/bin/echo selfcheck ok" +} diff --git a/conf.d/selfcheck.json b/conf.d/selfcheck.json deleted file mode 100644 index 56cec93..0000000 --- a/conf.d/selfcheck.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "services": [ - { - "name": "", - "group": "mesos" - } - ], - "conf_name": "selfcheck", - "template": "templates/selfcheck.jj2", - "dest": "selfcheck", - "reload_cmd": "/bin/echo selfcheck ok" -} diff --git a/conf/surok.json b/conf/surok.json index bed8ebc..f93bdb4 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,22 +1,27 @@ { + "version": "0.8", "marathon": { - "force": true, - "host": "http://marathon.mesos:8080", - "enabled": false - }, + "enabled": false, + "restart": false, + "force": true, + "host": "http://marathon.mesos:8080" + }, + "mesos": { + "enabled": false, + "domain": "marathon.mesos" + }, + "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", - "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", "loglevel": "info", - "container": false, - "memcached": { - "enabled": false, - "discovery": { - "enabled": false, - "service": "memcached", - "group": "system" - }, - "hosts": ["localhost:11211"] - } + "memcached": { + "enabled": false, + "discovery": { + "enabled": false, + "service": "memcached", + "group": "system" + }, + "hosts": ["localhost:11211"] + } } diff --git a/debian/changelog b/debian/changelog index 7d14ccc..7ce8512 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +surok (0.8) testing; urgency=medium + + * Version bump + * new config structure + * Port type + * Discovery over Marathon API + + -- Denis Zheleztsov Tue, 07 Feb 2017 12:41:52 +0300 + surok (0.7.4.3) testing; urgency=medium * Fixed #9 @@ -11,7 +20,7 @@ surok (0.7.4.3) testing; urgency=medium surok (0.7.4.1) testing; urgency=medium * closes #3 - * Fixed bug with very long dns query lifetime + * Fixed bug with very long dns query lifetime -- Denis Zheleztsov Fri, 11 Nov 2016 21:15:02 +0300 diff --git a/debian/install b/debian/install index 74eab76..d00a88b 100644 --- a/debian/install +++ b/debian/install @@ -1,7 +1,11 @@ conf/surok.json etc/surok/conf +conf.d/self_check.json etc/surok/conf.d +conf.d/marathon_check.json etc/surok/conf.d +templates/selfcheck.jj2 etc/surok/templates surok/templates.py opt/surok/surok surok/system.py opt/surok/surok surok/logger.py opt/surok/surok surok/__init__.py opt/surok/surok surok/discovery.py opt/surok/surok +surok/config.py opt/surok/surok surok.py opt/surok diff --git a/docs/App-config-file.html b/docs/App-config-file.html index 8c0bd6e..8b5c2bc 100644 --- a/docs/App-config-file.html +++ b/docs/App-config-file.html @@ -34,9 +34,9 @@ } * services. List of hashes with required services for app. - 1. name - string. App name in Marathon. + 1. name - string. App name in Marathon. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. 2. group - string. App group in Marathon. Optional. Discovery policy: 1) config 2) SUROKDISCOVERYGROUP environment variable 3) Marathon API - 3. ports - list. Name of opened port. In marathon of course. Optional. + 3. ports - list. Name of opened port. In marathon of course. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. Optional. * confname. Unique app config name. * template. Jinja2 template location. * dest. Destination config path. diff --git a/docs/App-config-file.md b/docs/App-config-file.md index 60b63bd..602b059 100644 --- a/docs/App-config-file.md +++ b/docs/App-config-file.md @@ -5,26 +5,27 @@ conf.d/myapp.json { "services": [ { - "name": "myapp", - "group": "backend.production", - "ports": ["proxy", "api"] - }, - { - "name": "nginx", - "group": "frontend.production" + "name": "kioskservice", + "group": "production.romania", + "ports": ["web", "socket"] } ], - "conf_name": "myapp_backend_production", - "template": "/etc/surok/templates/myapp.jj2", - "dest": "/etc/myapp/myapp.cfg", - "reload_cmd": "killall -9 myapp; /usr/local/bin/myapp -config /etc/myapp/myapp.cfg" + "discovery": "mesos_dns", + "group": "dev.web", + "conf_name": "kiosk", + "template": "/etc/surok/templates/kiosk.jj2", + "dest": "/etc/nginx/sites-available/kioskservice.conf", + "reload_cmd": "/sbin/nginx -t && /bin/systemctl reload nginx", + "run_cmd": ["/usr/bin/node", "-c", "config.json"] } ``` * **services**. List of hashes with required services for app. - 1. _name_ - string. App name in Marathon. + 1. _name_ - string. App name in Marathon. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. 2. _group_ - string. App group in Marathon. Optional. Discovery policy: 1) config 2) SUROK_DISCOVERY_GROUP environment variable 3) Marathon API - 3. _ports_ - list. Name of opened port. In marathon of course. Optional. + 3. _ports_ - list. Name of opened port. In marathon of course. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. Optional. * **conf_name**. Unique app config name. * **template**. Jinja2 template location. * **dest**. Destination config path. * **reload_cmd**. Command to execute if generated config is changed. +* **discovery**. Use custom discovery for app. +* **group**. Default group for all required services. diff --git a/docs/Main-config-file.html b/docs/Main-config-file.html index ff6c3da..e14bd5c 100644 --- a/docs/Main-config-file.html +++ b/docs/Main-config-file.html @@ -19,12 +19,21 @@ { "marathon": { - "force": true, - "host": "http://marathon.mesos:8080", - "enabled": false + "enabled": false, + "restart": false, + "force": true, + "host": "http://marathon.mesos:8080" }, + "consul": { + "enabled": false, + "domain": "service.dc1.consul" + }, + "mesos":{ + "enabled": true, + "domain": "marathon.mesos" + }, + "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", - "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", "loglevel": "info", diff --git a/docs/Main-config-file.md b/docs/Main-config-file.md index 431fb44..cd75b99 100644 --- a/docs/Main-config-file.md +++ b/docs/Main-config-file.md @@ -1,36 +1,82 @@ -# Surok main config file +# Surok main config file (0.8.x) -Default location is /etc/surok/conf/surok.json +Default location is **/etc/surok/conf/surok.json** -conf/surok.json ``` { + "version": "0.8" "marathon": { - "force": true, - "host": "http://marathon.mesos:8080", - "enabled": false - }, + "enabled": false, + "restart": false, + "force": true, + "host": "http://marathon.mesos:8080" + }, + "mesos":{ + "enabled": true, + "domain": "marathon.mesos" + }, + "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", - "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", "loglevel": "info", - "container": false, "memcached": { - "enabled": false, - "discovery": { - "enabled": false, - "service": "memcached", - "group": "system" - }, - "hosts": ["localhost:11211"] + "enabled": false, + "discovery": { + "enabled": false, + "service": "memcached", + "group": "system" + }, + "hosts": ["localhost:11211"] } } ``` -* **marathon section**. Restarting app over marathon api if config changed. Disabled by default. -* **confd**. Directory where located configs apps. -* **domain**. Domain served by mesos-dns. -* **lock_dir**. Directory where surok writes temporary configs after resolving. -* **wait_time**. Sleep time in main loop. -* **container**. Not implemented. -* **memcached section**. Memcached support. Disabled by default. + +## Config file options +* **version** - *string. Optional. "0.7" by default.* +Config files and templates version. Accept "0.7" or "0.8". + * "0.7" - config files <= 0.7.х version + * "0.8" - >= 0.8.x config files version + +##### 0.8 version +* **marathon**, **mesos**, **consul**, **memcached** - *dict/hash. Optional. '{"enable":false}'. by default* +Surok working with folowing systems. If system is disabled parameters will be ignored. + * **enable** - *boolean. Optional. false by default* + Enable/disable system for usage. + + Specific variables: + * For Marathon API "marathon" + * **force** - *boolean. Optional. true by default* + Force restart container over API. + * **restart** - *boolean. Optional. false by default* + Enable/disable restarting container + * **host** - *string. Optional. "http://marathon.mesos:8080" by default* + Marathon address. + * For mesos-dns "mesos" + * **domain** - *string. Optional. "marathon.mesos" by default* + mesos-dns private domain + * For Memcached "memcached" + * **hosts** - memcached hosts + * **discovery** + * **enabled** - boolean. Enable/disable disovery memcached service + * **service** - string. memcached app name + * **group** - string. memcached app group +* **default_discovery** - *string. Optional. "mesos_dns" by default* + Accept values: + * "mesos_dns" - mesos-dns + * "marathon_api"- Marathon API +* **confd** - *strig. Required.* + Path to directory with app config files. +* **wait_time** - *int. Required.* + Time in seconds how much Surok waits before starting to re-do the requests for service discovery +* **lock_dir** - *string. Required.* + Path to directory where Surok write lock-files. +* **loglevel** - *string. Optional. "info" by default* + Logleve. Accept values: "debug", "info", "warning", "error" + +##### < 0.8 versions + +* **marathon** + * **enabled** - boolean. Enable/disable container restart. Renamed to "restart" in 0.8 version. +* **domain** - string. mesos-dns private domain. Moved to "mesos" hashtable in 0.8 version. + Discovery over mesos-dns enabled all times. diff --git a/docs/README.md b/docs/README.md index 82774d8..a5815d2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,6 +31,7 @@ Service discovery for Apache Mesos. * Denis Zheleztsov * Denis Ryabyy +* Evgeniy Vasilev ## LICENSE diff --git a/docs/Templates.md b/docs/Templates.md index 50ea7d5..90d2431 100644 --- a/docs/Templates.md +++ b/docs/Templates.md @@ -5,8 +5,11 @@ - [Templates](#templates) - [my dictionary in templates](#my-dictionary-in-templates) - - [Real template example](#real-template-example) - - [Checks in template](#checks-in-template) + - [0.8 version](#08-version) + - [Real template example](#real-template-example) + - [Checks in template](#checks-in-template) + - [0.7 version](#07-version) + - [Real template example](#real-template-example) @@ -15,55 +18,74 @@ Surok using Jinja2 for templates. [Jinja2 documentation](http://jinja.pocoo.org/ ## my dictionary in templates +### 0.8 version + ``` { "services": { - "nginx": [ + "asterisk": [ { - "name": "nginx.testing-kl92-s0.marathon.mesos.", - "port": "31200", - "ip": ["10.10.10.1"] + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1", + "11.0.0.1" + ], + "tcp": { + "rpc":31200, + "web":31201, + "sip":32000 + }, + "udp": { + "sip":31201 + } }, { - "name": "nginx.testing-kl123-s1.marathon.mesos.", - "port": "32230", - "ip": ["10.10.10.2"] + "name": "nginx.testing-kl123-s1.marathon.mesos.", + "ip": [ + "10.0.0.2", + "11.0.0.2" + ], + "tcp": { + "rpc":31210, + "web":31211, + "sip":32010 + }, + "udp": { + "sip":31211 + } } ], - "emailsender": [ + "email": [ { - "name": "emailsender.testing-kl92-s0.marathon.mesos.", - "port": "31201", - "ip": ["10.10.10.1"] - }, - { - "name": "emailsender.testing-kl123-s1.marathon.mesos.", - "port": "32232", - "ip": ["10.10.10.1"] + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1" + ], + "tcp": { + "smtp":31200, + "pop":31201 + } } ], - "service-with-defined-ports": { - "web": [ - { - "name": "f.q.d.n", - "port": 12341 - } - ], - "rpc": [ - { - "name": "f.q.d.n", - "port": 12342 - } - ] - } - }, + "anyport": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1" + ], + "tcp": [ + 31200, + 31201 + ] + } + ] "env": { "HOME": "/var/lib/nginx" } } ``` -## Real template example +#### Real template example nginx config ``` @@ -104,7 +126,7 @@ server { } ``` -## Checks in template +#### Checks in template _my['env']_ is a python os.environ class. Look bellow: ``` @@ -114,3 +136,90 @@ host = '{{my['env']['DB_HOST']}}' host = 'localhost' {% endif %} ``` + +### 0.7 version + +my dictionary in template +``` +{ + "services": { + "nginx": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "port": "31200" + }, + { + "name": "nginx.testing-kl123-s1.marathon.mesos.", + "port": "32230" + } + ], + "emailsender": [ + { + "name": "emailsender.testing-kl92-s0.marathon.mesos.", + "port": "31201" + }, + { + "name": "emailsender.testing-kl123-s1.marathon.mesos.", + "port": "32232" + } + ], + "service-with-defined-ports": { + "name-of-port0": [ + { + "name": "f.q.d.n", + "port": 12341 + } + ], + "name-of-port2": [ + { + "name": "f.q.d.n", + "port": 12342 + } + ] + } + }, + "env": { + "HOME": "/var/lib/nginx" + } +} +``` + +#### Real template example + +``` +upstream matrix-http { + hash $remote_addr; +{% for server in my['services']['matrix']['http'] %} + server {{server['name']}}:{{server['port']}} max_fails=3; +{% endfor %} +} + +upstream riot-http { + hash $remote_addr; +{% for server in my['services']['riot'] %} + server {{server['name']}}:{{server['port']}} max_fails=3; +{% endfor %} +} + +server { + listen 10.15.56.157:80; + server_name matrix.example.com; + + client_max_body_size 10m; + + location / { + proxy_pass http://riot-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /_matrix/ { + proxy_pass http://matrix-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + +} +``` diff --git a/docs/ru/app.configuration.md b/docs/ru/app.configuration.md index 11c89d6..5dcba65 100644 --- a/docs/ru/app.configuration.md +++ b/docs/ru/app.configuration.md @@ -10,6 +10,8 @@ "ports": ["web", "socket"] } ], + "discovery": "mesos_dns", + "group": "dev.web", "conf_name": "kiosk", "template": "/etc/surok/templates/kiosk.jj2", "dest": "/etc/nginx/sites-available/kioskservice.conf", @@ -31,3 +33,5 @@ В reload_cmd можно использовать переменные окружения: ```"reload_cmd": "/usr/bin/killall -9 calc || true && /usr/local/bin/calc -c /app/calc.conf ${CALC_NUM}"``` * run_cmd(v0.6) - array. Список с командой на выполнение. Используется внутри контейнера вместо reload_cmd. +* discovery - Опциональный. переопределяет метод обнаружения сервисов для приложения +* group - Опциональный. Группа по-умолчанию. diff --git a/docs/ru/surok.configuration.md b/docs/ru/surok.configuration.md index 213ae15..a52283b 100644 --- a/docs/ru/surok.configuration.md +++ b/docs/ru/surok.configuration.md @@ -1,30 +1,81 @@ -# Конфигурация Surok (0.7.x) - -**/etc/surok/conf/surok.json** -Разберем конфигурационный файл по опциям +# Конфигурация Surok (0.8.x) +**/etc/surok/conf/surok.json** Разберем конфигурационный файл по опциям ``` { + "version": "0.8" "marathon": { - "force": true, - "host": "marathon.mesos:8080", - "enabled": true + "enabled": false, + "restart": false, + "force": true, + "host": "http://marathon.mesos:8080" }, + "mesos":{ + "enabled": true, + "domain": "marathon.mesos" + }, + "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", - "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", - "loglevel": "info|debug" - "container": true|false + "loglevel": "info", + "memcached": { + "enabled": false, + "discovery": { + "enabled": false, + "service": "memcached", + "group": "system" + }, + "hosts": ["localhost:11211"] + } } ``` +## Опции файла конфигурации +* **version** - *string. Не обязательный. По умолчанию "0.7".* +Версия файлов конфигурации, шаблонов. На текущий момент может принимать значения "0.7" или "0.8". + * значение "0.7" - файлы конфигурации версии 0.7.х и более ранних + * значение "0.8" - файлы конфигурации версии 0.8 + +##### версия 0.8 +* **marathon**, **mesos**, **consul**, **memcached** - *dict/hash. Не обязательный. По умолчанию '{"enable":false}'.* +Системы с которыми работает сурок. Если система выключена, то параметры системы и их наличие уже не важны. + * **enable** - *boolean. Не обязательный. По умолчанию false.* + Доступность системы для использования. + + специфичные параметры: + * для Marathon API "marathon" + * **force** - *boolean. Не обязательный. По умолчанию true.* + Рестарт контейнера с force или нет. + * **restart** - *boolean. Не обязательный. По умолчанию false.* + Вкл/выкл. рестарта контейнера + * **host** - *string. Не обязательный. По умолчанию "http://marathon.mesos:8080".* + Адрес Marathon. + * для mesos DNS "mesos" + * **domain** - *string. Не обязательный. По умолчанию "marathon.mesos".* + Приватный домен Mesos DNS + * для Memcached "memcached" + * **hosts** - + * **discovery** - + * **enabled** - + * **service** - + * **group** - +* **default_discovery** - *string. Не обязательный. По умолчанию "mesos_dns".* + + Может принимать значения: + * "mesos_dns" - Mesos DNS + * "marathon_api"- Marathon API +* **confd** - *strig. Обязательный.* + Абсолютный путь до директории с конфигурационными файлами приложений. +* **wait_time** - *int. Обязательный.* + Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов. +* **lock_dir** - *string. Обязательный.* + Абсолютный путь до директории с lock-конфигурациями. +* **loglevel** - *string. Не обязательный. По умолчанию "info".* + Уровень логирования. Может принимать значения: "debug", "info", "warning", "error" + +##### версия 0.7 и более ранние +Особенности для файла конфигурации +* **marathon** + * **enabled** - boolean. Вкл/выкл. рестарта контейнера. В версии 0.8 переименована в "restart". +* **domain** - string. Приватный домен Mesos DNS. В версии 0.8 перемещен в dict "mesos". + Обнаружение Mesos DNS включено всегда. -* marathon(v0.7) - hash. В текущей версии отвечает за перезапуск контейнера. Обнаружение сервисов через Marathon пока недоступно. - 1. force - boolean. Рестарт контейнера с force или нет. - 2. host - string. Адрес Marathon. - 3. enabled - boolean. Вкл/выкл. -* confd - strig. Абсолютный путь до директории с конфигурационными файлами приложений. -* domain - string. Домен, который обслуживает mesos-dns. -* wait_time - int. Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов. -* lock_dir - string. Абсолютный путь до директории с lock-конфигурациями. -* loglevel - string. Уровень логирования. -* container(v0.6) - boolean. Определяем внутри или нет контейнера запущен сурок. Меняется логика работы. diff --git a/docs/ru/templates.md b/docs/ru/templates.md index 4b98f32..c7609a5 100644 --- a/docs/ru/templates.md +++ b/docs/ru/templates.md @@ -4,6 +4,127 @@ ## Словарь my в шаблоне +### Версия 0.8 +Surok заполняет словарь my и передает его в шаблон. +``` +{ + "services": { + "asterisk": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1", + "11.0.0.1" + ], + "tcp": { + "rpc":31200, + "web":31201, + "sip":32000 + }, + "udp": { + "sip":31201 + } + }, + { + "name": "nginx.testing-kl123-s1.marathon.mesos.", + "ip": [ + "10.0.0.2", + "11.0.0.2" + ], + "tcp": { + "rpc":31210, + "web":31211, + "sip":32010 + }, + "udp": { + "sip":31211 + } + } + ], + "email": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1" + ], + "tcp": { + "smtp":31200, + "pop":31201 + } + } + ], + "anyport": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1" + ], + "tcp": [ + 31200, + 31201 + ] + } + ] + "env": { + "HOME": "/var/lib/nginx" + } +} +``` + +## Пример реального шаблона + +``` +upstream matrix-http { + hash $remote_addr; +{% for server in my['services']['matrix'] %} + server {{server['name']}}:{{server['tcp']['http']}} max_fails=3; +{% endfor %} +} + +upstream riot-http { + hash $remote_addr; +{% for server in my['services']['riot'] %} + server {{server['name']}}:{{server['tcp'][0]}} max_fails=3; +{% endfor %} +} + +server { + listen 10.15.56.157:80; + server_name matrix.example.com; + + client_max_body_size 10m; + + location / { + proxy_pass http://riot-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /_matrix/ { + proxy_pass http://matrix-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + +} +``` +Так для upstream matrix-http используются именованные порты, а для riot-http – нет. + +## Проверки в шаблоне + +Переменная _my['env']_ является классом python _os.environ_, что позваоляет нам строить различные проверки, например: + +``` +{% if my['env'].get('DB_HOST') -%} +host = '{{my['env']['DB_HOST']}}' +{% else -%} +host = 'localhost' +{% endif -%} +``` + +### Версия 0.7 Surok заполняет словарь my и передает его в шаблон. ``` { @@ -29,13 +150,13 @@ Surok заполняет словарь my и передает его в шаб } ], "service-with-defined-ports": { - "web": [ + "name-of-port0": [ { "name": "f.q.d.n", "port": 12341 } ], - "rpc": [ + "name-of-port2": [ { "name": "f.q.d.n", "port": 12342 @@ -58,34 +179,34 @@ upstream matrix-http { server {{server['name']}}:{{server['port']}} max_fails=3; {% endfor %} } - + upstream riot-http { hash $remote_addr; {% for server in my['services']['riot'] %} server {{server['name']}}:{{server['port']}} max_fails=3; {% endfor %} } - + server { listen 10.15.56.157:80; server_name matrix.example.com; - + client_max_body_size 10m; - + location / { proxy_pass http://riot-http; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - + location /_matrix/ { proxy_pass http://matrix-http; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - + } ``` Так для upstream matrix-http используются именованные порты, а для riot-http – нет. diff --git a/surok.py b/surok.py index 31f3dc3..8ea7fb0 100755 --- a/surok.py +++ b/surok.py @@ -7,71 +7,42 @@ from os.path import isfile, join import json import argparse from surok.templates import gen -from surok.discovery import resolve +from surok.discovery import Discovery from surok.system import reload_conf +from surok.logger import Logger +from surok.config import Config - -# Load base configurations -surok_conf = '/etc/surok/conf/surok.json' +logger = Logger() # Command line arguments parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', help='surok.json path') args = parser.parse_args() -if args.config: - surok_conf = args.config - -# Read config file -f = open(surok_conf, 'r') -conf = json.loads(f.read()) -f.close() - - -# Get app configurations -# Return list of patches to app discovery configuration -def get_configs(): - confs = [f for f in listdir(conf['confd']) if isfile( - join(conf['confd'], f))] - return sorted(confs) - - -# Get Surok App configuration -# Read app conf from file and return dict -def load_app_conf(app): - # Load OS environment to app_conf - f = open(conf['confd'] + '/' + app) - c = json.loads(f.read()) - f.close() - - c['env'] = os.environ - - return c +# Load base configurations +config = Config(args.config if args.config else '/etc/surok/conf/surok.json') # Main loop -########### +# + +discovery = Discovery() while 1: - confs = get_configs() - for app in confs: - app_conf = load_app_conf(app) + # Update config from discovery object + discovery.update_data() + for app in config.apps: - # Will be removed later - # For old configs - loglevel = 'info' - if 'loglevel' in conf: - loglevel = conf['loglevel'] - - # Resolve services - app_hosts = resolve(app_conf, conf) + app_hosts = discovery.resolve(app) # Populate my dictionary my = {"services": app_hosts, - "conf_name": app_conf['conf_name']} + "conf_name": app['conf_name']} + + logger.debug('my=', my) # Generate config from template - service_conf = gen(my, app_conf['template']) + service_conf = gen(my, app['template']) - reload_conf(service_conf, app_conf, conf, app_hosts) + reload_conf(service_conf, app, config, app_hosts) - sleep(conf['wait_time']) + sleep(config['wait_time']) diff --git a/surok.spec b/surok.spec index 82289d1..763b6a1 100644 --- a/surok.spec +++ b/surok.spec @@ -1,6 +1,6 @@ Summary: Simple service discovery for Apache Mesos clusters Name: surok -Version: 0.7.4.3 +Version: 0.8 Release: 1 License: BSD Group: admin @@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates} install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf -install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/requriments.txt %{buildroot}/opt/surok @@ -59,7 +61,11 @@ cd /opt/surok && pip3 install -r requriments.txt /opt/surok/surok/templates.py /opt/surok/surok/templates.pyc /opt/surok/surok/templates.pyo -/etc/surok/conf.d/selfcheck.json +/opt/surok/surok/config.py +/opt/surok/surok/config.pyc +/opt/surok/surok/config.pyo +/etc/surok/conf.d/self_check.json +/etc/surok/conf.d/marathon_check.json /etc/surok/templates/selfcheck.jj2 /opt/surok/requriments.txt %defattr(-,root,root,-) @@ -67,6 +73,9 @@ cd /opt/surok && pip3 install -r requriments.txt %changelog +* Tue Feb 7 2017 Denis Zheleztsov +- New major release +- Discovery over marathon api * Mon Nov 14 2016 Denis Zheleztsov - - Initial build. diff --git a/surok/config.py b/surok/config.py new file mode 100644 index 0000000..9e73ef4 --- /dev/null +++ b/surok/config.py @@ -0,0 +1,420 @@ +# Public names +__all__ = ['Config', 'AppConfig'] + +import hashlib +import importlib +import json +import os +from .logger import * + +# Logger link + +# Config singleton link +_config_singleton = None + +''' +Test values +================================================== +key - key +value - value of key +type_value - type of value +type_par - additional parameters for test + +''' + + +''' +Public Config object +================================================== +.set_config(conf_data) - set config data + Use: conf_data(str type) - path of json config file + conf_data(dict type) - dict with config +.set(key,value) - set config key +.get(key) - get config key +.update_apps() - update apps config data +.apps - Apps object. List of AppConfig oblects +''' + + +class _ConfigTemplate(dict): + _conf = {} + + def _init_conf(self, params): + conf = {} + for k in params.keys(): + if params[k].get('params'): + conf[k] = self._init_conf(params[k].get('params')) + else: + if params[k].get('value') is not None: + conf[k] = params[k].get('value') + return conf + + def __init__(self, *conf_data): + self._logger = Logger() + if not self._conf: + self._conf = self._init_conf(self._params) + for c in conf_data: + self.set_config(c) + + def _set_conf_params(self, oldconf, testconf, params): + conf = oldconf if oldconf else {} + for key in testconf.keys(): + resvalue = None + param = params.get(key) + oldvalue = conf.get(key) + testvalue = testconf.get(key) + if param is None: + self._logger.error('Parameter "', key, '" value "', testvalue, + '" type is "', type(testvalue).__name__, '" not found') + else: + type_param = param.get('type') + resvalue = [] + if type(testvalue).__name__ != 'list': + testvalue = [testvalue] + for testitem in testvalue: + if self._test_value(key, testitem, param): + if 'dict' in type_param: + if param.get('params'): + res = self._set_conf_params( + oldvalue, testitem, param.get('params')) + if res is not None: + resvalue.append(res) + else: + resvalue.append(testitem) + if 'list' not in type_param: + resvalue = list([None] + resvalue).pop() + if resvalue is not None and 'do' in type_param: + if not self._do_type_set(key, resvalue, param): + self._logger.warning( + 'Parameter "', key, '" current "', resvalue, '" type is "', type(resvalue).__name__, '" testing failed') + resvalue = None + if resvalue is not None: + conf[key] = resvalue + return conf + + def _test_value(self, key, value, param): + type_param = param.get('type') + type_value = [ + x for x in type_param if x in ['str', 'int', 'bool', 'dict']] + if type_value: + if type(value).__name__ not in type_value: + self._logger.error( + 'Parameter "', key, '" must be ', type_value, + ' types, current "', value, '" (', type(value).__name__, ')') + return False + if 'value' in type_param: + if value not in param.get('values', []): + self._logger.error( + 'Value "', value, '" of key "', key, '" unknown') + return False + if 'dir' in type_param: + if not os.path.isdir(value): + self._logger.error('Path "{}" not present'.format(value)) + return False + elif 'file' in type_param: + if not os.path.isfile(value): + self._logger.error('File "{}" not present'.format(value)) + return False + return True + else: + self._logger.error( + 'Type for testing "{}" unknown'.format(type_value)) + return False + + def set_config(self, conf_data): + conf = {} + if type(conf_data).__name__ == 'str': + try: + self._logger.debug('Open file ', conf_data) + f = open(conf_data, 'r') + json_data = f.read() + f.close() + conf = json.loads(json_data) + except OSError as err: + self._logger.error("OS error: {0}".format(err)) + pass + except ValueError as err: + self._logger.error('JSON format error: {0}'.format(err)) + pass + except: + self._logger.error('Load main config file failed') + pass + elif type(conf).__name__ == 'dict': + conf = conf_data + else: + return False + self._conf = self._set_conf_params(self._conf, conf, self._params) + self._logger.debug('Conf=', self._conf) + + def keys(self): + return self._conf.keys() + + def dump(self): + return json.dumps(self._conf, sort_keys=True, indent=2) + + def _do_type_set(self, key, value, params): + self._logger.error('_do_type_set handler is not defined') + return False + + def hash(self): + return hashlib.sha1(json.dumps(self._conf, sort_keys=True).encode()).hexdigest() + + def set(self, key, value): + self._conf[key] = value + + def __setitem__(self, key, value): + self.set(key, value) + + def get(self, key, default=None): + return self._conf.get(key, default) + + def __getitem__(self, key): + return self.get(key) + + def __contains__(self, item): + return bool(item in self._conf) + + def __len__(self): + return self._conf.__len__() + + def __str__(self): + return self._conf.__str__() + + def __repr__(self): + return self._conf.__repr__() + + +class Config(_ConfigTemplate): + _params = { + 'marathon': { + 'params': { + 'force': { + 'value': True, + 'type': ['bool'] + }, + 'host': { + 'value': 'http://marathon.mesos:8080', + 'type': ['str'] + }, + 'enabled': { + 'value': False, + 'type': ['bool'] + }, + 'restart': { + 'value': False, + 'type': ['bool'] + } + }, + 'type': ['dict'] + }, + 'mesos': { + 'params': { + "domain": { + 'value': "marathon.mesos", + 'type': ['str'] + }, + "enabled": { + 'value': False, + 'type': ['bool'] + } + }, + 'type': ['dict'] + }, + 'memcached': { + 'params': { + 'enabled': { + 'value': False, + 'type': ['bool'] + }, + 'discovery': { + 'params': { + 'enabled': { + 'value': False, + 'type': ['bool'] + }, + 'service': { + 'value': 'memcached', + 'type': ['str'] + }, + 'group': { + 'value': 'system', + 'type': ['str'] + } + }, + 'type': ['dict'] + }, + 'hosts': { + 'value': ['localhost:11211'], + 'type': ['list', 'str'] + } + }, + 'type': ['dict'] + }, + 'version': { + 'value': '0.7', + 'type': ['str', 'value'], + 'values': ['0.7', '0.8'] + }, + 'confd': { + 'value': '/etc/surok/conf.d', + 'type': ['str', 'dir'] + }, + 'wait_time': { + 'value': 20, + 'type': ['int'] + }, + 'lock_dir': { + 'value': '/var/tmp', + 'type': ['str', 'dir'] + }, + 'default_discovery': { + 'value': 'mesos_dns', + 'type': ['str', 'value'], + 'values': ['mesos_dns', 'marathon_api'] + }, + 'loglevel': { + 'value': 'info', + 'type': ['str', 'do'], + 'do': 'set_loglevel' + } + } + + def __new__(cls, *args): + global _config_singleton + if _config_singleton is None: + _config_singleton = super(Config, cls).__new__(cls) + return _config_singleton + + def __init__(self, *conf_data): + super().__init__(*conf_data) + self.apps = _Apps() + + def set_config(self, conf_data): + super().set_config(conf_data) + if self.get('version') == '0.7': + domain = self.get('domain') + if domain is not None and self.get('mesos') is None: + self.set('mesos', {'domain': domain, 'enabled': True}) + + def _do_type_set(self, key, value, param): + if param.get('do') == 'set_loglevel': + if self._logger.set_level(value): + return True + return False + + def update_apps(self): + self.apps.reset() + for app in sorted([os.path.join(self.get('confd'), f) for f in os.listdir(self.get('confd')) if os.path.isfile(os.path.join(self.get('confd'), f))]): + self.apps.set(AppConfig(app)) + +''' +Private Apps object +================================================== +.get(app) - get _AppConfig object +.set(app) - set _AppConfig object +''' + + +class _Apps: + _apps = {} + _items = [] + + def get(self, key): + return self._apps.get(key) + + def set(self, app): + self._apps[app.get('conf_name')] = app + + def reset(self): + keys = [] + list(self.keys()) + for k in keys: + del self._apps[k] + + def __getitem__(self, key): + return self.get(key) + + def __contains__(self, item): + return bool(item in self._apps) + + def __iter__(self): + self._items = sorted(self.keys()) + return self + + def __next__(self): + if self._items: + return self.get(self._items.pop(0)) + raise StopIteration + + def keys(self): + return self._apps.keys() + +''' +Public AppConfig object +================================================== +.set_config(conf_data) - set config data + Use: conf_data(str type) - path of json config file + conf_data(dict type) - dict with config +.set(key,value) - set config key +.get(key) - get config key +''' + + +class AppConfig(_ConfigTemplate): + _params = { + 'conf_name': { + 'type': ['str'] + }, + 'services': { + 'params': { + 'name': { + 'type': ['str'] + }, + 'ports': { + 'type': ['list', 'str'] + }, + 'discovery': { + 'type': ['str'] + }, + 'group': { + 'type': ['str'] + } + }, + 'type': ['list', 'dict'] + }, + 'template': { + 'type': ['str', 'file'] + }, + 'dest': { + 'type': ['str'] + }, + 'reload_cmd': { + 'type': ['str'] + }, + 'discovery': { + 'type': ['str'] + }, + 'group': { + 'type': ['str'] + } + } + + def __init__(self, *conf_data): + self._config = Config() + super().__init__(*conf_data) + + def set_config(self, conf_data): + super().set_config(conf_data) + self._conf.setdefault( + 'discovery', self._config.get('default_discovery')) + self._conf.setdefault('group', self._get_default_group()) + if type(conf_data).__name__ == 'str': + self._conf.setdefault('conf_name', os.path.basename(conf_data)) + + def _get_default_group(self): + env = self._config.get('env', dict(os.environ)) + # Check environment variable + if env.get('SUROK_DISCOVERY_GROUP'): + return env['SUROK_DISCOVERY_GROUP'] + # Check marathon environment variable + elif env.get('MARATHON_APP_ID'): + return ".".join(env['MARATHON_APP_ID'].split('/')[-2:0:-1]) diff --git a/surok/discovery.py b/surok/discovery.py index 4f93da5..25bad3c 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -1,95 +1,232 @@ +# Public names +__all__ = ['Discovery', 'DiscoveryMesos', 'DiscoveryMarathon'] import dns.resolver import dns.query -from dns.exception import DNSException -from .logger import info, warning, error, debug +import os import sys +import requests +from dns.exception import DNSException +from .config import * +from .logger import * + +# Discovery object +_discovery_singleton = None -# Resolve service from mesos-dns SRV record -# return dict {"servicename": [{"name": "service.f.q.d.n.", "port": 9999}]} -def resolve(app, conf): - hosts = {} - services = app['services'] - domain = conf['domain'] +class DiscoveryTemplate: - for service in services: - hosts[service['name']] = {} + def __init__(self): + self._config = Config() + self._logger = Logger() - group = get_group(service, app) - if group is False: - error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID') - error('Not in Mesos launch?') - sys.exit(2) + def enabled(self): + return self._config[self._config_section].get('enabled', False) - # Port name from app config - ports = None + def update_data(self): + pass + + # Do DNS queries + # Return array: + # ["10.10.10.1", "10.10.10.2"] + def do_query_a(self, fqdn): + servers = [] try: - ports = service['ports'] + resolver = dns.resolver.Resolver() + for a_rdata in resolver.query(fqdn, 'A'): + servers.append(a_rdata.address) + except DNSException as e: + self._logger.error("Could not resolve ", fqdn) + return servers + + # Do DNS queries + # Return array: + # [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}] + def do_query_srv(self, fqdn): + servers = [] + try: + resolver = dns.resolver.Resolver() + resolver.lifetime = 1 + resolver.timeout = 1 + query = resolver.query(fqdn, 'SRV') + for rdata in query: + info = str(rdata).split() + servers.append({'name': info[3][:-1], 'port': info[2]}) + except DNSException as e: + self._logger.warning("Could not resolve ", fqdn) + return servers + + +class Discovery: + _discoveries = {} + + def __new__(cls): + global _discovery_singleton + if _discovery_singleton is None: + _discovery_singleton = super(Discovery, cls).__new__(cls) + return _discovery_singleton + + def __init__(self): + self._config = Config() + self._logger = Logger() + if not self._discoveries.get('mesos_dns'): + self._discoveries['mesos_dns'] = DiscoveryMesos() + if not self._discoveries.get('marathon_api'): + self._discoveries['marathon_api'] = DiscoveryMarathon() + + def keys(self): + return self._discoveries.keys() + + def resolve(self, app): + discovery = app.get('discovery', self._config.get('default_discovery')) + if discovery not in self.keys(): + self._logger.warning('Discovery "', discovery, '" is not present') + return {} + if self._discoveries[discovery].enabled(): + return self.compatible(self._discoveries[discovery].resolve(app)) + else: + self._logger.error('Discovery "', discovery, '" is disabled') + return {} + + def update_data(self): + self._config.update_apps() + for d in self.keys(): + if self._discoveries[d].enabled(): + self._discoveries[d].update_data() + + def compatible(self, hosts): + compatible_hosts = {} + if self._config.get('version') == '0.7': + for service in hosts.keys(): + for host in hosts[service]: + ports = host.get('tcp', []) + if type(ports).__name__ == 'list': + compatible_hosts[service] = [] + for port in ports: + compatible_hosts[service].append( + {'name': host['name'], + 'ip': host['ip'], + 'port': str(port)}) + else: + compatible_hosts[service] = {} + for port in ports.keys(): + compatible_host = compatible_hosts[ + service].setdefault(port, []) + compatible_host.append({'name': host['name'], + 'ip': host['ip'], + 'port': ports[port]}) + + return compatible_hosts + return hosts + + +class DiscoveryMesos(DiscoveryTemplate): + _config_section = 'mesos' + + def resolve(self, app): + hosts = {} + services = app.get('services') + domain = self._config[self._config_section].get('domain') + for service in services: + group = service.get('group', app.get('group')) + if group is None: + self._logger.error('Group for service "{}" of config "{}" not found'.format( + service['name'], app.get('conf_name'))) + continue + ports = service.get('ports') + name = service['name'] + hosts[name] = {} + serv = hosts[name] + self._logger.debug( + 'group=', group, ' ports=', ports, ' name=', name, ' serv=', serv) + for prot in ['tcp', 'udp']: + if ports is not None: + for port_name in ports: + for d in self.do_query_srv('_' + port_name + '._' + name + '.' + group + '._' + prot + '.' + domain): + hostname = d['name'] + serv.setdefault(hostname, {'name': hostname, + 'ip': self.do_query_a(hostname)}) + serv[hostname].setdefault(prot, {}) + serv[hostname][prot][port_name] = d['port'] + else: + for d in self.do_query_srv('_' + name + '.' + group + '._' + prot + '.' + domain): + hostname = d['name'] + if serv.get(hostname) is None: + serv[hostname] = {'name': hostname, + 'ip': self.do_query_a(hostname)} + if serv[hostname].get(prot) is None: + serv[hostname][prot] = [] + serv[hostname][prot].extend([d['port']]) + hosts[name] = list(serv.values()) + return hosts + + +class DiscoveryMarathon(DiscoveryTemplate): + _config_section = 'marathon' + _tasks = [] + _ports = {} + + def update_data(self): + hostname = self._config[self._config_section].get('host') + try: + ports = {} + for app in requests.get(hostname + '/v2/apps').json()['apps']: + ports[app['id']] = {} + if app.get('container') is not None and app['container'].get('type') == 'DOCKER': + ports[app['id']] = app['container'][ + 'docker'].get('portMappings', []) + self._ports = ports except: + self._logger.warning( + 'Apps (', hostname, '/v2/apps) request from Marathon API is failed') + pass + try: + self._tasks = requests.get(hostname + '/v2/tasks').json()['tasks'] + except: + self._logger.warning( + 'Tasks (', hostname, '/v2/tasks) request from Marathon API is failed') pass - # This is fast fix for port naming - # Will be rewrite later - fqdn = '' - if ports is not None: - for port_name in ports: - fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain - hosts[service['name']][port_name] = do_query(fqdn, conf['loglevel']) - else: - fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain - hosts[service['name']] = do_query(fqdn, conf['loglevel']) + def _test_mask(self, mask, value): + return (mask.endswith('*') and value.startswith(mask[:-1])) or mask == value - return hosts - - -# Do DNS queries -# Return array: -# [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}] -def do_query(fqdn, loglevel): - servers = [] - try: - resolver = dns.resolver.Resolver() - resolver.lifetime = 1 - resolver.timeout = 1 - query = resolver.query(fqdn, 'SRV') - for rdata in query: - info = str(rdata).split() - name = info[3][:-1] - port = info[2] - server = {'name': name, 'port': port, 'ip': []} - a_query = resolver.query(name, 'A') - for a_rdata in a_query: - server['ip'].append(a_rdata.address) - servers.append(server) - except DNSException as e: - if loglevel != 'info': - error("Could not resolve " + fqdn) - - return servers - - -# Groups switch -# Priority: config, environment, marathon environment -def get_group(service, app): - # Check group in app conf - if 'group' in service: - return service['group'] - # Check environment variable - elif app['env'].get('SUROK_DISCOVERY_GROUP'): - return app['env']['SUROK_DISCOVERY_GROUP'] - # Check marathon environment variable - elif app['env'].get('MARATHON_APP_ID'): - group = parse_marathon_app_id(app['env']['MARATHON_APP_ID']) - return group - else: - return False - - -# Parse MARATHON_APP_ID -# Return marathon.group -def parse_marathon_app_id(marathon_app_id): - marathon_app_id = marathon_app_id.split('/') - del(marathon_app_id[-1]) - marathon_app_id.reverse() - group = ".".join(marathon_app_id)[:-1] - return(group) + def resolve(self, app): + hosts = {} + services = app.get('services') + if not services: + services = [{'name': '*', 'ports': ['*']}] + for service in services: + # Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format + group = service.get('group', app.get('group')) + if group is None: + self._logger.error('Group for service "{}" of config "{}" not found'.format( + service['name'], app.get('conf_name'))) + continue + group = '/' + '/'.join(group.split('.')[::-1]) + '/' + service_mask = group + service['name'] + for task in self._tasks: + if self._test_mask(service_mask, task['appId']): + name = '.'.join( + task['appId'][len(group):].split('/')[::-1]) + hosts[name] = {} + serv = hosts[name] + hostname = task['host'] + for task_port in self._ports[task['appId']]: + prot = task_port['protocol'] + port_name = task_port['name'] + port = task['ports'][ + task['servicePorts'].index(task_port['servicePort'])] + if 'ports' in service: + for port_mask in service['ports']: + if self._test_mask(port_mask, port_name): + serv.setdefault( + hostname, {'name': hostname, + 'ip': self.do_query_a(hostname)}) + serv[hostname].setdefault(prot, {}) + serv[hostname][prot][port_name] = port + else: + serv.setdefault(hostname, {'name': hostname, + 'ip': self.do_query_a(hostname)}) + serv[hostname].setdefault(prot, []) + serv[hostname][prot].extend([port]) + hosts[name] = list(serv.values()) + return hosts diff --git a/surok/logger.py b/surok/logger.py index e9baf26..1bf2fa6 100644 --- a/surok/logger.py +++ b/surok/logger.py @@ -1,36 +1,86 @@ +# Public names +__all__ = ['Logger'] + import sys -from time import time +import json +import time + +# Logger singleton link +_logger_singleton = None + +''' +Public Logger oblect +================================================== +.set_level(level) - set level messages + level - values 'debug', 'info', 'warning', 'error' + * error - write error message + * warning - write warning and error message + * info - write info, warning and error message + * debug - write all message +.get_level() - get level messages +.error(str) - write error message +.warning(str) - write warning message +.info(str) - write info message +.debug(str)- write error message +''' -def make_message(message): - cur_time = str(time()) - m = '[' + cur_time + '] ' + message['level'] + ': ' + message['raw'] + "\n" - return m +class Logger: + _loglevel = 'info' + _msg_level = { + 'debug': 'DEBUG', + 'info': 'INFO', + 'warning': 'WARNING', + 'error': 'ERROR' + } + def __new__(cls, *args): + global _logger_singleton + if _logger_singleton is None: + _logger_singleton = super(Logger, cls).__new__(cls) + return _logger_singleton -def info(message): - req = {'level': 'INFO', 'raw': message} - m = make_message(req) + def __init__(self, *args): + if args: + self.set_level(args[0]) - sys.stdout.write(m) + def set_level(self, level): + if level in ['debug', 'info', 'warning', 'error']: + self._loglevel = level + return True + else: + self.warning('Log level "', level, '" not valid') + return False + def get_level(self): + return self._loglevel -def warning(message): - req = {'level': 'WARNING', 'raw': message} - m = make_message(req) + def _make_message(self, level, message): + r = [] + for m in message: + if type(m).__name__ == 'str': + r.append(m) + else: + r.append(json.dumps(m, sort_keys=True, indent=2)) + return '[' + str(time.time()) + '] ' + self._msg_level[level] + ': ' + ''.join(r) + "\n" - sys.stderr.write(m) + def debug(self, *message): + if self.get_level() in ['debug']: + self._log2err(self._make_message('debug', message)) + def info(self, *message): + if self.get_level() in ['debug', 'info']: + self._log2out(self._make_message('info', message)) -def error(message): - req = {'level': 'ERROR', 'raw': message} - m = make_message(req) + def warning(self, *message): + if self.get_level() in ['debug', 'info', 'warning']: + self._log2out(self._make_message('warning', message)) - sys.stderr.write(m) + def error(self, *message): + self._log2err(self._make_message('error', message)) + def _log2err(self, out): + sys.stderr.write(out) -def debug(message): - req = {'level': 'DEBUG', 'raw': message} - m = make_message(req) - - sys.stderr.write(m) + def _log2out(self, out): + sys.stdout.write(out) diff --git a/surok/system.py b/surok/system.py index 324961b..81717b5 100644 --- a/surok/system.py +++ b/surok/system.py @@ -1,20 +1,20 @@ import os import sys import requests -from .discovery import resolve -from .logger import info, warning, error, debug +from .discovery import Discovery +from .logger import Logger +logger=Logger() # Get old configuration def get_old(name, service_conf): - try: path = '/var/tmp/surok.' + name f = open(path, 'r') old = f.read() f.close() except Exception as e: - print(str(e)) + logger.error(str(e)) return 0 if old == service_conf: @@ -53,7 +53,7 @@ def write_lock(name, service_conf): def do_reload(service_conf, app_conf): - warning('Write new configuration of ' + app_conf['conf_name']) + logger.warning('Write new configuration of ' + app_conf['conf_name']) f = open(app_conf['dest'], 'w') f.write(service_conf) @@ -68,6 +68,7 @@ def do_reload(service_conf, app_conf): # Discovery memcached servers def discovery_memcached(conf): + discovery=Discovery() memcache = conf['memcached'] app_conf = { "services": [ @@ -78,7 +79,7 @@ def discovery_memcached(conf): ] } - hosts = resolve(app_conf, conf) + hosts = discovery.resolve(app_conf) mc_servers = [] for server in hosts[memcache['discovery']['service']]: @@ -91,7 +92,8 @@ def discovery_memcached(conf): # !!! NEED REFACTORING !!! def reload_conf(service_conf, app_conf, conf, app_hosts): # Check marathon enabled in configuration - if conf['marathon']['enabled'] is True: + if (conf.get('version','0.7')=='0.8' and conf['marathon'].get('restart',False)) or ( + conf.get('version','0.7')=='0.7' and conf['marathon'].get('enable',False)): if get_old(app_conf['conf_name'], service_conf) != 1: restart_self_in_marathon(conf['marathon']) @@ -105,47 +107,38 @@ def reload_conf(service_conf, app_conf, conf, app_hosts): mc_hosts = None if conf['memcached']['discovery']['enabled'] is True: mc_hosts = discovery_memcached(conf) - info('Discovered memcached hosts: ' + str(mc_hosts)) + logger.info('Discovered memcached hosts: ' + str(mc_hosts)) else: mc_hosts = conf['memcached']['hosts'] try: mc = memcache.Client(mc_hosts) if get_old_from_memcache(mc, app_conf['conf_name'], app_hosts) != 1: stdout = do_reload(service_conf, app_conf) - info(stdout) + logger.info(stdout) return True except Exception as e: - error('Cannot connect to memcached: ' + str(e)) + logger.error('Cannot connect to memcached: ' + str(e)) else: - warning('DEPRECATED main conf file. Please use new syntax!') + logger.warning('DEPRECATED main conf file. Please use new syntax!') # End of memcache block ####################### if get_old(app_conf['conf_name'], service_conf) != 1: stdout = do_reload(service_conf, app_conf) - info(stdout) + logger.info(stdout) return True else: - if conf['loglevel'] == 'debug': - debug('Same config ' + app_conf['conf_name'] + ' Skip reload') + logger.debug('Same config ' + app_conf['conf_name'] + ' Skip reload') return False - # Do POST request to marathon API # /v2/apps//app/name/restart def restart_self_in_marathon(marathon): - host = marathon['host'] - # Check MARATHON_APP_ID environment varible - if os.environ.get('MARATHON_APP_ID') is not True: - error('Cannot find MARATHON_APP_ID. Not in Mesos?') + if not os.environ.get('MARATHON_APP_ID',False): + logger.error('Cannot find MARATHON_APP_ID. Not in Mesos?') sys.exit(2) - app_id = os.environ['MARATHON_APP_ID'] - uri = 'http://' + host + '/v2/apps/' + app_id + '/restart' - # Ok. In this step we made restart request to Marathon - if marathon['force'] is True: - r = requests.post(uri, data = {'force': 'true'}) - else: - r = requests.post(uri, data = {'force': 'false'}) + r = requests.post('http://'+marathon['host']+'/v2/apps/'+os.environ['MARATHON_APP_ID']+'/restart', + data={'force': marathon.get('force',False)}) diff --git a/surok_fedora.spec b/surok_fedora.spec index 57f839f..d9cbad6 100644 --- a/surok_fedora.spec +++ b/surok_fedora.spec @@ -1,6 +1,6 @@ Summary: Simple service discovery for Apache Mesos clusters Name: surok -Version: 0.7.4.3 +Version: 0.8 Release: 1.fc24 License: BSD Group: admin @@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates} install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf -install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates @@ -43,13 +45,18 @@ rm -rf $RPM_BUILD_ROOT /opt/surok/surok/discovery.py /opt/surok/surok/system.py /opt/surok/surok/templates.py -/etc/surok/conf.d/selfcheck.json +/opt/surok/surok/config.py +/etc/surok/conf.d/self_check.json +/etc/surok/conf.d/marathon_check.json /etc/surok/templates/selfcheck.jj2 %defattr(-,root,root,-) %doc %changelog +* Tue Feb 7 2017 Denis Zheleztsov +- New major release +- Discovery over marathon api * Mon Nov 14 2016 Denis Zheleztsov - - Initial build.