From 19ed189cc3c6f8e80d8ac07f5abd3c58e5701427 Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Wed, 18 Jan 2017 10:11:09 +0300 Subject: [PATCH 01/10] POC realization of #15 --- surok/discovery.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/surok/discovery.py b/surok/discovery.py index 4f93da5..9dc3cfd 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -13,7 +13,7 @@ def resolve(app, conf): domain = conf['domain'] for service in services: - hosts[service['name']] = {} + hosts[service['name']] = [] group = get_group(service, app) if group is False: @@ -23,18 +23,38 @@ def resolve(app, conf): # Port name from app config ports = None - try: + if 'ports' in service: ports = service['ports'] - except: - pass - # This is fast fix for port naming - # Will be rewrite later + # "service-with-defined-ports": + # [ + # { + # "name": "example1.com", + # "ip": ["10.10.10.1"], + # "ports": { + # "rpc": 12342, + # "web": 12341 + # } + # }, + # { + # "name": "example2.com", + # "ports": { + # "rpc": 12344, + # "web": 12343 + # } + # } + # ] fqdn = '' if ports is not None: for port_name in ports: - fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain - hosts[service['name']][port_name] = do_query(fqdn, conf['loglevel']) + fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain # Need support for udp ports. See #16 + discovered = do_query(fqdn, conf['loglevel']) + for d in discovered: + to_append = {} + to_append['name'] = d['name'] + to_append['ip'] = d['ip'] + to_append['ports'][port_name] = d['port'] + hosts[service['name']].append(to_append) else: fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain hosts[service['name']] = do_query(fqdn, conf['loglevel']) From 3511f1318096a5f5e901039a02c4d795e4b455ea Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Sat, 21 Jan 2017 10:58:02 +0300 Subject: [PATCH 02/10] Consul DNS discovery. Related to #12 --- conf/surok.json | 32 ++++++++++++++++++-------------- surok/discovery.py | 9 ++++++++- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/conf/surok.json b/conf/surok.json index bed8ebc..c48ef84 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,22 +1,26 @@ { "marathon": { - "force": true, - "host": "http://marathon.mesos:8080", - "enabled": false - }, + "force": true, + "host": "http://marathon.mesos:8080", + "enabled": false + }, + "consul": { + "enabled": false, + "domain": "service.dc1.consul" + }, "confd": "/etc/surok/conf.d", "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", "loglevel": "info", - "container": false, - "memcached": { - "enabled": false, - "discovery": { - "enabled": false, - "service": "memcached", - "group": "system" - }, - "hosts": ["localhost:11211"] - } + "container": false, + "memcached": { + "enabled": false, + "discovery": { + "enabled": false, + "service": "memcached", + "group": "system" + }, + "hosts": ["localhost:11211"] + } } diff --git a/surok/discovery.py b/surok/discovery.py index 4f93da5..1e78730 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -14,7 +14,7 @@ def resolve(app, conf): for service in services: hosts[service['name']] = {} - + group = get_group(service, app) if group is False: error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID') @@ -31,6 +31,13 @@ def resolve(app, conf): # This is fast fix for port naming # Will be rewrite later fqdn = '' + + # Discovery over Consul DNS + if 'consul' in conf and conf['consul']['enabled']: + fqdn = '_' + service['name'] + '._tcp.' + conf['consul']['domain'] + hosts[service['name']] = do_query(fqdn, conf['loglevel']) + continue + if ports is not None: for port_name in ports: fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain From 12b868ad0be7ff2223f5333a122b3c5ffdfc50b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=92=D0=BB=D0=B0=D0=B4?= =?UTF-8?q?=D0=B8=D0=BC=D0=B8=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 25 Jan 2017 13:11:51 +0300 Subject: [PATCH 03/10] Add Marathon discovery. Remade discovery and logger module --- README.ru.md | 19 ++- build/tests.py | 53 +++--- conf/surok.json | 35 ++-- docs/App-config-file.html | 4 +- docs/App-config-file.md | 4 +- docs/Main-config-file.html | 17 +- docs/Main-config-file.md | 33 ++-- surok.py | 25 ++- surok/discovery.py | 340 +++++++++++++++++++++++++++---------- surok/logger.py | 63 ++++--- surok/system.py | 45 ++--- 11 files changed, 430 insertions(+), 208 deletions(-) diff --git a/README.ru.md b/README.ru.md index 022ddc4..66adf3f 100644 --- a/README.ru.md +++ b/README.ru.md @@ -1,5 +1,5 @@ # Surok -[![Build Status](https://travis-ci.org/Difrex/surok.svg?branch=master)](https://travis-ci.org/Difrex/surok) +[![Build Status](https://travis-ci.org/Surkoveds/surok.svg?branch=master)](https://travis-ci.org/Surkoveds/surok) Обнаружение сервисов для Apache Mesos. @@ -17,10 +17,27 @@ cd build deb-пакет будет лежать в build/out Сборка базового docker-образа surok +Ubuntu Xenial ``` cd build ./build.sh surok_image ``` +Alpine image +``` +cd build +./build.sh alpine +``` +CentOS image +``` +cd build +./build.sh centos +``` + +ENTRYPOINT : ```cd /opt/surok && pytho3 surok.py -c /etc/surok/conf/surok.json``` + +## Документация + +[Wiki](https://github.com/Surkoveds/surok/wiki) ## Известные проблемы diff --git a/build/tests.py b/build/tests.py index ece5c7b..3828fe4 100644 --- a/build/tests.py +++ b/build/tests.py @@ -3,7 +3,6 @@ import json import os import re - class TestLoadConfig(unittest.TestCase): def test_main_conf(self): @@ -16,7 +15,6 @@ class TestLoadConfig(unittest.TestCase): self.assertIn('confd', conf) self.assertTrue(os.path.isdir(conf['confd'])) - self.assertIn('domain', conf) self.assertIn('wait_time', conf) self.assertIn('lock_dir', conf) self.assertTrue(os.path.isdir(conf['lock_dir'])) @@ -24,48 +22,59 @@ class TestLoadConfig(unittest.TestCase): class TestLogger(unittest.TestCase): + def test_debug(self): + from surok.logger import Logger + m = Logger() + self.assertIn('DEBUG', m.testing('debug','log message')) + def test_info(self): - from surok.logger import make_message - m = make_message - self.assertIn('INFO', m({'level': 'INFO', 'raw': 'log message'})) + from surok.logger import Logger + m = Logger() + self.assertIn('INFO', m.testing('info','log message')) def test_warning(self): - from surok.logger import make_message - m = make_message - self.assertIn('WARNING', m({'level': 'WARNING', 'raw': 'log message'})) + from surok.logger import Logger + m = Logger() + self.assertIn('WARNING', m.testing('warning','log message')) def test_error(self): - from surok.logger import make_message - m = make_message - self.assertIn('ERROR', m({'level': 'ERROR', 'raw': 'log message'})) - - def test_info(self): - from surok.logger import make_message - m = make_message - self.assertIn('DEBUG', m({'level': 'DEBUG', 'raw': 'log message'})) + from surok.logger import Logger + m = Logger() + self.assertIn('ERROR', m.testing('error','log message')) class TestMemcachedDiscovery(unittest.TestCase): def test_discovery_memcache(self): from surok.system import discovery_memcached - + from surok.discovery import Discovery # Load base configurations surok_conf = '/etc/surok/conf/surok.json' # Read config file f = open(surok_conf, 'r') conf = json.loads(f.read()) f.close() - + d=Discovery(conf) self.assertEqual(discovery_memcached(conf), []) class TestGetGroup(unittest.TestCase): - def test_get_group(self): - from surok.discovery import get_group - self.assertFalse(get_group({}, {'env': os.environ})) - + def test_get_group_from_service(self): + from surok.discovery import DiscoveryTemplate + d=DiscoveryTemplate({}) + self.assertEqual('xxx.yyy.zzz',d.get_group({'group':'xxx.yyy.zzz'}, {})) + + def test_get_group_from_env(self): + from surok.discovery import DiscoveryTemplate + d=DiscoveryTemplate({}) + self.assertEqual('xxx.yyy.zzz',d.get_group({}, {'env':{'SUROK_DISCOVERY_GROUP':'xxx.yyy.zzz'}})) + + def test_get_group_from_marathon_id(self): + from surok.discovery import DiscoveryTemplate + d=DiscoveryTemplate({}) + self.assertEqual('xxx.yyy.zzz',d.get_group({}, {'env':{'MARATHON_APP_ID':'/zzz/yyy/xxx/www'}})) + if __name__ == '__main__': unittest.main() diff --git a/conf/surok.json b/conf/surok.json index c48ef84..e1c58f2 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,26 +1,31 @@ { "marathon": { - "force": true, - "host": "http://marathon.mesos:8080", - "enabled": false - }, + "enabled": false, + "restart": false, + "force": true, + "host": "http://marathon.mesos:8080" + }, "consul": { "enabled": false, "domain": "service.dc1.consul" }, + "mesos":{ + "enabled": true, + "domain": "marathon.mesos" + }, + "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", - "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", "loglevel": "info", - "container": false, - "memcached": { - "enabled": false, - "discovery": { - "enabled": false, - "service": "memcached", - "group": "system" - }, - "hosts": ["localhost:11211"] - } + "container": false, + "memcached": { + "enabled": false, + "discovery": { + "enabled": false, + "service": "memcached", + "group": "system" + }, + "hosts": ["localhost:11211"] + } } diff --git a/docs/App-config-file.html b/docs/App-config-file.html index 8c0bd6e..8b5c2bc 100644 --- a/docs/App-config-file.html +++ b/docs/App-config-file.html @@ -34,9 +34,9 @@ } * services. List of hashes with required services for app. - 1. name - string. App name in Marathon. + 1. name - string. App name in Marathon. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. 2. group - string. App group in Marathon. Optional. Discovery policy: 1) config 2) SUROKDISCOVERYGROUP environment variable 3) Marathon API - 3. ports - list. Name of opened port. In marathon of course. Optional. + 3. ports - list. Name of opened port. In marathon of course. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. Optional. * confname. Unique app config name. * template. Jinja2 template location. * dest. Destination config path. diff --git a/docs/App-config-file.md b/docs/App-config-file.md index 60b63bd..5156e5a 100644 --- a/docs/App-config-file.md +++ b/docs/App-config-file.md @@ -21,9 +21,9 @@ conf.d/myapp.json } ``` * **services**. List of hashes with required services for app. - 1. _name_ - string. App name in Marathon. + 1. _name_ - string. App name in Marathon. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. 2. _group_ - string. App group in Marathon. Optional. Discovery policy: 1) config 2) SUROK_DISCOVERY_GROUP environment variable 3) Marathon API - 3. _ports_ - list. Name of opened port. In marathon of course. Optional. + 3. _ports_ - list. Name of opened port. In marathon of course. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. Optional. * **conf_name**. Unique app config name. * **template**. Jinja2 template location. * **dest**. Destination config path. diff --git a/docs/Main-config-file.html b/docs/Main-config-file.html index ff6c3da..e14bd5c 100644 --- a/docs/Main-config-file.html +++ b/docs/Main-config-file.html @@ -19,12 +19,21 @@ { "marathon": { - "force": true, - "host": "http://marathon.mesos:8080", - "enabled": false + "enabled": false, + "restart": false, + "force": true, + "host": "http://marathon.mesos:8080" }, + "consul": { + "enabled": false, + "domain": "service.dc1.consul" + }, + "mesos":{ + "enabled": true, + "domain": "marathon.mesos" + }, + "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", - "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", "loglevel": "info", diff --git a/docs/Main-config-file.md b/docs/Main-config-file.md index 431fb44..db2efd6 100644 --- a/docs/Main-config-file.md +++ b/docs/Main-config-file.md @@ -6,24 +6,33 @@ conf/surok.json ``` { "marathon": { - "force": true, - "host": "http://marathon.mesos:8080", - "enabled": false - }, + "enabled": false, + "restart": false, + "force": true, + "host": "http://marathon.mesos:8080" + }, + "consul": { + "enabled": false, + "domain": "service.dc1.consul" + }, + "mesos":{ + "enabled": true, + "domain": "marathon.mesos" + }, + "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", - "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", "loglevel": "info", "container": false, "memcached": { - "enabled": false, - "discovery": { - "enabled": false, - "service": "memcached", - "group": "system" - }, - "hosts": ["localhost:11211"] + "enabled": false, + "discovery": { + "enabled": false, + "service": "memcached", + "group": "system" + }, + "hosts": ["localhost:11211"] } } ``` diff --git a/surok.py b/surok.py index 31f3dc3..4429e19 100755 --- a/surok.py +++ b/surok.py @@ -7,10 +7,11 @@ from os.path import isfile, join import json import argparse from surok.templates import gen -from surok.discovery import resolve +from surok.discovery import Discovery from surok.system import reload_conf +from surok.logger import Logger - +logger=Logger() # Load base configurations surok_conf = '/etc/surok/conf/surok.json' @@ -47,28 +48,34 @@ def load_app_conf(app): return c +logger.set_level(conf.get('loglevel','info')) # Main loop ########### +discovery=Discovery() + while 1: confs = get_configs() + + # Update config from discovery object + discovery.set_config(conf) + + # Update discovery data + discovery.update_data() + for app in confs: app_conf = load_app_conf(app) - # Will be removed later - # For old configs - loglevel = 'info' - if 'loglevel' in conf: - loglevel = conf['loglevel'] - # Resolve services - app_hosts = resolve(app_conf, conf) + app_hosts = discovery.resolve(app_conf) # Populate my dictionary my = {"services": app_hosts, "conf_name": app_conf['conf_name']} + logger.debug('my=',my) + # Generate config from template service_conf = gen(my, app_conf['template']) diff --git a/surok/discovery.py b/surok/discovery.py index ff494fa..429e51b 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -1,78 +1,270 @@ import dns.resolver import dns.query from dns.exception import DNSException -from .logger import info, warning, error, debug +from .logger import Logger import sys +import requests +# Default config for Discovery class +_config={ + 'default_discovery':'mesos_dns' # Default discovery system +} -# Resolve service from mesos-dns SRV record -# return dict {"servicename": [{"name": "service.f.q.d.n.", "port": 9999}]} -def resolve(app, conf): - hosts = {} - services = app['services'] - domain = conf['domain'] +# Discoveries objects +_discoveries={} - for service in services: - hosts[service['name']] = [] +#Logger +logger=Logger() - group = get_group(service, app) - if group is False: - error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID') - error('Not in Mesos launch?') +class DiscoveryTemplate: + # Default config values for discovery template + _config={} + _defconfig={'enabled':False} + + def __init__(self,conf): + for key in self._defconfig.keys(): + if key not in self._config.keys(): + self._config[key]=self._defconfig[key] + self.set_config(conf) + + def set_config(self,conf): + pass + + def enabled(self): + return self._config['enabled'] + + def update_data(self): + pass + + def get_group(self,service, app): + # Check group in app conf + if 'group' in service: + return service['group'] + + # Check environment variable + elif app['env'].get('SUROK_DISCOVERY_GROUP'): + return app['env']['SUROK_DISCOVERY_GROUP'] + + # Check marathon environment variable + elif app['env'].get('MARATHON_APP_ID'): + return ".".join(app['env']['MARATHON_APP_ID'].split('/')[-2:0:-1]) + + else: + logger.error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID') + logger.error('Not in Mesos launch?') sys.exit(2) - # Port name from app config - ports = None - if 'ports' in service: - ports = service['ports'] - # "service-with-defined-ports": - # [ - # { - # "name": "example1.com", - # "ip": ["10.10.10.1"], - # "ports": { - # "rpc": 12342, - # "web": 12341 - # } - # }, - # { - # "name": "example2.com", - # "ports": { - # "rpc": 12344, - # "web": 12343 - # } - # } - # ] - fqdn = '' +class Discovery: + def __init__(self,*conf): + for __conf in conf: + self.set_config(__conf) - # Discovery over Consul DNS - if 'consul' in conf and conf['consul']['enabled']: - fqdn = '_' + service['name'] + '._tcp.' + conf['consul']['domain'] - hosts[service['name']].append(do_query(fqdn, conf['loglevel'])) - continue - - if ports is not None: - for port_name in ports: - fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain # Need support for udp ports. See #16 - discovered = do_query(fqdn, conf['loglevel']) - for d in discovered: - to_append = {} - to_append['name'] = d['name'] - to_append['ip'] = d['ip'] - to_append['ports'][port_name] = d['port'] - hosts[service['name']].append(to_append) + def set_config(self,conf): + global _discoveries + #Get discoveries objects + if not _discoveries.get('mesos_dns'): + _discoveries['mesos_dns']=DiscoveryMesos(conf) else: - fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain - hosts[service['name']] = do_query(fqdn, conf['loglevel']) + _discoveries['mesos_dns'].set_config(conf) - return hosts + if not _discoveries.get('marathon_api'): + _discoveries['marathon_api']=DiscoveryMarathon(conf) + else: + _discoveries['marathon_api'].set_config(conf) + + if not _discoveries.get('consul_dns'): + _discoveries['consul_dns']=DiscoveryConsul(conf) + else: + _discoveries['consul_dns'].set_config(conf) + + global _config + if conf.get('default_discovery'): + discovery=conf.get('default_discovery') + if discovery in list(_discoveries.keys()): + _config['default_discovery']=discovery + else: + logger.error('Default discovery "'+discovery+'" is not present') + logger.debug('Conf=',conf) + + def resolve(self,app): + __discovery=_config.get('default_discovery') + if app.get('discovery'): + discovery=app.get('discovery') + if discovery in list(_discoveries.keys()): + __discovery=discovery + else: + logger.warning('Discovery "'+discovery+'" is not present') + logger.debug('App=',app) + return {} + if _discoveries[__discovery].enabled(): + return _discoveries[__discovery].resolve(app) + else: + logger.error('Discovery "'+__discovery+'" is disabled') + return {} + + def update_data(self): + global _discoveries + for d in list(_discoveries.keys()): + if _discoveries[d].enabled(): + _discoveries[d].update_data() + + +class DiscoveryMesos(DiscoveryTemplate): + _config={ + 'domain':'marathon.mesos' # Default domain + } + + def set_config(self,conf): + # For old version config + if conf.get('domain'): + self._config['domain']=conf.get('domain') + self._config['enabled']=True + # For current version config + if conf.get('mesos'): + _conf=conf['mesos'] + for p in ['domain','enabled']: + if _conf.get(p): + self._config[p]=_conf.get(p) + + def resolve(self,app): + hosts = {} + services = app['services'] + domain = self._config['domain'] + for service in services: + group = self.get_group(service, app) + ports = service.get('ports') + name = service['name'] + hosts[name] = {} + serv = hosts[name] + if ports is not None: + hosts[name] = {} + serv = hosts[name] + for prot in ['tcp','udp']: + for port_name in ports: + for d in do_query('_'+port_name+'._'+name+'.'+group+'._'+prot+'.'+domain): + hostname=d['name'] + if serv.get(hostname) is None: + serv[hostname]={"name":hostname,"ip":d['ip']} + if serv[hostname].get(prot) is None: + serv[hostname][prot]={} + serv[hostname][prot][port_name]=d['port'] + hosts[name]=list(hosts[name].values()) + else: + hosts[name]=do_query('_'+name+'.'+group+'._tcp.'+domain) + + return hosts + + +class DiscoveryMarathon(DiscoveryTemplate): + _config={ + 'host':'http://marathon.mesos:8080', + 'force':True + } + __tasks = [] + __ports = {} + def set_config(self,conf): + # For current version config + if conf.get('marathon'): + _conf=conf['marathon'] + for p in ['host','enabled','force']: + if _conf.get(p): + self._config[p]=_conf.get(p) + + def update_data(self): + try: + apps = requests.get(self._config['host']+'/v2/apps').json()['apps'] + ports = {} + for app in apps: + ports[app['id']] = {} + if app.get('container') is not None and app['container']['type'] == 'DOCKER': + ports[app['id']] = app['container']['docker'].get('portMappings',[]) + self.__ports=ports + except: + logger.warning('Apps ('+self._config['host']+'/v2/apps) request from Marathon API is failed') + pass + try: + self.__tasks = requests.get(self._config['host']+'/v2/tasks').json()['tasks'] + except: + logger.warning('Tasks ('+self._config['host']+'/v2/tasks) request from Marathon API is failed') + pass + + def resolve(self, app): + hosts={} + serv_conf = app['services'] + if not serv_conf: + serv_conf = [{'name':'*','ports':['*']}] + for serv in serv_conf: + # Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format + group = '/'.join(['']+self.get_group(serv, app).split('.')[::-1]+['']) + mask = group+serv['name'] + for task in self.__tasks: + if (mask.endswith('*') and task['appId'].startswith(mask[:-1])) or task['appId'] == mask: + name='.'.join(task['appId'][len(group):].split('/')[::-1]) + if 'ports' in serv: + hosts[name]={} + for port in self.__ports[task['appId']]: + for pp in serv['ports']: + if (pp.endswith('*') and port['name'].startswith(pp[:-1])) or port['name'] == pp: + if hosts[name].get(task['host']) is None: + hosts[name][task['host']]={'name':task['host'], + 'ip':do_query_a(task['host'])} + if hosts[name][task['host']].get(port['protocol']) is None: + hosts[name][task['host']][port['protocol']]={} + hosts[name][task['host']][port['protocol']][port['name']]=task['ports'][task['servicePorts'].index(port['servicePort'])] + hosts[name]=list(hosts[name].values()) + else: + hosts[name]=[] + for port in self.__ports[task['appId']]: + hosts[name].append({'name':task['host'], + 'port':task['ports'][task['servicePorts'].index(port['servicePort'])], + 'ip':do_query_a(task['host'])}) + + return hosts + + +class DiscoveryConsul(DiscoveryTemplate): + _config={ + 'enabled':False, + 'domain':None + } + def set_config(self,conf): + # For current version config + if conf.get('consul'): + _conf=conf['consul'] + for p in ['domain','enabled']: + if _conf.get(p): + self._config[p]=_conf.get(p) + + def resolve(self,app): + hosts = {} + services = app['services'] + domain = self._config['domain'] + for service in services: + name = service['name'] + hosts[name]=do_query('_'+name+'._tcp.'+domain) + return hosts + + +# Do DNS queries +# Return array: +# ["10.10.10.1", "10.10.10.2"] +def do_query_a(fqdn): + servers = [] + try: + resolver = dns.resolver.Resolver() + for a_rdata in resolver.query(fqdn, 'A'): + servers.append(a_rdata.address) + except DNSException as e: + logger.error("Could not resolve "+fqdn) + + return servers # Do DNS queries # Return array: # [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}] -def do_query(fqdn, loglevel): +def do_query(fqdn): servers = [] try: resolver = dns.resolver.Resolver() @@ -83,40 +275,8 @@ def do_query(fqdn, loglevel): info = str(rdata).split() name = info[3][:-1] port = info[2] - server = {'name': name, 'port': port, 'ip': []} - a_query = resolver.query(name, 'A') - for a_rdata in a_query: - server['ip'].append(a_rdata.address) - servers.append(server) + servers.append({'name': name, 'port': port, 'ip': do_query_a(name)}) except DNSException as e: - if loglevel != 'info': - error("Could not resolve " + fqdn) + logger.error("Could not resolve " + fqdn) return servers - - -# Groups switch -# Priority: config, environment, marathon environment -def get_group(service, app): - # Check group in app conf - if 'group' in service: - return service['group'] - # Check environment variable - elif app['env'].get('SUROK_DISCOVERY_GROUP'): - return app['env']['SUROK_DISCOVERY_GROUP'] - # Check marathon environment variable - elif app['env'].get('MARATHON_APP_ID'): - group = parse_marathon_app_id(app['env']['MARATHON_APP_ID']) - return group - else: - return False - - -# Parse MARATHON_APP_ID -# Return marathon.group -def parse_marathon_app_id(marathon_app_id): - marathon_app_id = marathon_app_id.split('/') - del(marathon_app_id[-1]) - marathon_app_id.reverse() - group = ".".join(marathon_app_id)[:-1] - return(group) diff --git a/surok/logger.py b/surok/logger.py index e9baf26..7cee349 100644 --- a/surok/logger.py +++ b/surok/logger.py @@ -1,36 +1,51 @@ import sys +import json from time import time +_loglevel='info' +msg_level={'debug':'DEBUG', + 'info':'INFO', + 'warning':'WARNING', + 'error':'ERROR'} +class Logger: + def __init__(self,*args): + if args: + self.set_level(args[0]) -def make_message(message): - cur_time = str(time()) - m = '[' + cur_time + '] ' + message['level'] + ': ' + message['raw'] + "\n" - return m + def set_level(self,level): + if level in ['debug','info','warning','error']: + global _loglevel + _loglevel=level + def get_level(self): + return _loglevel -def info(message): - req = {'level': 'INFO', 'raw': message} - m = make_message(req) + def __make_message(self,message): + r=[] + l=self.get_level() + for m in message: + if type(m).__name__=='str': + r.append(m) + else: + r.append(json.dumps(m,sort_keys=True,indent=2)) + return '[' + str(time()) + '] ' + msg_level[l] + ': ' + ''.join(r) + "\n" - sys.stdout.write(m) + def debug(self,*message): + if self.get_level() in ['debug']: + sys.stderr.write(self.__make_message(message)) + def info(self,*message): + if self.get_level() in ['debug','info']: + sys.stdout.write(self.__make_message(message)) -def warning(message): - req = {'level': 'WARNING', 'raw': message} - m = make_message(req) + def warning(self,*message): + if self.get_level() in ['debug','info','warning']: + sys.stderr.write(self.__make_message(message)) - sys.stderr.write(m) + def error(self,*message): + sys.stderr.write(self.__make_message(message)) + def testing(self,level,message): + self.set_level(level) + return self.__make_message(message) -def error(message): - req = {'level': 'ERROR', 'raw': message} - m = make_message(req) - - sys.stderr.write(m) - - -def debug(message): - req = {'level': 'DEBUG', 'raw': message} - m = make_message(req) - - sys.stderr.write(m) diff --git a/surok/system.py b/surok/system.py index 324961b..89b1466 100644 --- a/surok/system.py +++ b/surok/system.py @@ -1,20 +1,19 @@ import os import sys import requests -from .discovery import resolve -from .logger import info, warning, error, debug - +from .discovery import Discovery +from .logger import Logger +logger=Logger() # Get old configuration def get_old(name, service_conf): - try: path = '/var/tmp/surok.' + name f = open(path, 'r') old = f.read() f.close() except Exception as e: - print(str(e)) + logger.error(str(e)) return 0 if old == service_conf: @@ -53,7 +52,7 @@ def write_lock(name, service_conf): def do_reload(service_conf, app_conf): - warning('Write new configuration of ' + app_conf['conf_name']) + logger.warning('Write new configuration of ' + app_conf['conf_name']) f = open(app_conf['dest'], 'w') f.write(service_conf) @@ -68,6 +67,7 @@ def do_reload(service_conf, app_conf): # Discovery memcached servers def discovery_memcached(conf): + discovery=Discovery() memcache = conf['memcached'] app_conf = { "services": [ @@ -78,7 +78,7 @@ def discovery_memcached(conf): ] } - hosts = resolve(app_conf, conf) + hosts = discovery.resolve(app_conf) mc_servers = [] for server in hosts[memcache['discovery']['service']]: @@ -91,7 +91,7 @@ def discovery_memcached(conf): # !!! NEED REFACTORING !!! def reload_conf(service_conf, app_conf, conf, app_hosts): # Check marathon enabled in configuration - if conf['marathon']['enabled'] is True: + if conf['marathon'].get('restart',False): if get_old(app_conf['conf_name'], service_conf) != 1: restart_self_in_marathon(conf['marathon']) @@ -105,47 +105,38 @@ def reload_conf(service_conf, app_conf, conf, app_hosts): mc_hosts = None if conf['memcached']['discovery']['enabled'] is True: mc_hosts = discovery_memcached(conf) - info('Discovered memcached hosts: ' + str(mc_hosts)) + logger.info('Discovered memcached hosts: ' + str(mc_hosts)) else: mc_hosts = conf['memcached']['hosts'] try: mc = memcache.Client(mc_hosts) if get_old_from_memcache(mc, app_conf['conf_name'], app_hosts) != 1: stdout = do_reload(service_conf, app_conf) - info(stdout) + logger.info(stdout) return True except Exception as e: - error('Cannot connect to memcached: ' + str(e)) + logger.error('Cannot connect to memcached: ' + str(e)) else: - warning('DEPRECATED main conf file. Please use new syntax!') + logger.warning('DEPRECATED main conf file. Please use new syntax!') # End of memcache block ####################### if get_old(app_conf['conf_name'], service_conf) != 1: stdout = do_reload(service_conf, app_conf) - info(stdout) + logger.info(stdout) return True else: - if conf['loglevel'] == 'debug': - debug('Same config ' + app_conf['conf_name'] + ' Skip reload') + logger.debug('Same config ' + app_conf['conf_name'] + ' Skip reload') return False - # Do POST request to marathon API # /v2/apps//app/name/restart def restart_self_in_marathon(marathon): - host = marathon['host'] - # Check MARATHON_APP_ID environment varible - if os.environ.get('MARATHON_APP_ID') is not True: - error('Cannot find MARATHON_APP_ID. Not in Mesos?') + if not os.environ.get('MARATHON_APP_ID',False): + logger.error('Cannot find MARATHON_APP_ID. Not in Mesos?') sys.exit(2) - app_id = os.environ['MARATHON_APP_ID'] - uri = 'http://' + host + '/v2/apps/' + app_id + '/restart' - # Ok. In this step we made restart request to Marathon - if marathon['force'] is True: - r = requests.post(uri, data = {'force': 'true'}) - else: - r = requests.post(uri, data = {'force': 'false'}) + r = requests.post('http://'+marathon['host']+'/v2/apps/'+os.environ['MARATHON_APP_ID']+'/restart', + data={'force': marathon.get('force',False)}) From 1f09875aa3421173d18a9f7846f3e2d23e63e975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=92=D0=BB=D0=B0=D0=B4?= =?UTF-8?q?=D0=B8=D0=BC=D0=B8=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 7 Feb 2017 01:39:30 +0300 Subject: [PATCH 04/10] Update russian docs. Add compatible for version 0.7. Add config module. Fix discovery module. Update test.py --- AUTHORS | 1 + build/runtests.sh | 2 +- build/tests.py | 426 +++++++++++++++++++++++++++------ conf.d/marathon_check.json | 12 + conf.d/self_check.json | 17 ++ conf.d/selfcheck.json | 12 - conf/surok.json | 8 +- debian/install | 4 + docs/README.md | 1 + docs/ru/surok.configuration.md | 99 ++++++-- docs/ru/templates.md | 137 ++++++++++- surok.py | 54 +---- surok.spec | 10 +- surok/config.py | 410 +++++++++++++++++++++++++++++++ surok/discovery.py | 385 +++++++++++++---------------- surok/logger.py | 97 +++++--- surok/system.py | 4 +- surok_fedora.spec | 8 +- 18 files changed, 1267 insertions(+), 420 deletions(-) mode change 100644 => 100755 build/tests.py create mode 100644 conf.d/marathon_check.json create mode 100644 conf.d/self_check.json delete mode 100644 conf.d/selfcheck.json create mode 100644 surok/config.py diff --git a/AUTHORS b/AUTHORS index a720639..354c3b8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,2 +1,3 @@ Denis Zheleztsov Denis Ryabyy +Evgeniy Vasilev diff --git a/build/runtests.sh b/build/runtests.sh index 37cbbe7..f54968f 100755 --- a/build/runtests.sh +++ b/build/runtests.sh @@ -5,7 +5,7 @@ set -e . functions.sh function run_tests() { - docker run -ti -v $(pwd)/tests.py:/opt/surok/tests.py \ + docker run --rm -ti -v $(pwd)/tests.py:/opt/surok/tests.py \ -v $(pwd)/tests_entrypoint.sh:/tests_entrypoint.sh \ --entrypoint /tests_entrypoint.sh \ surok_base:latest diff --git a/build/tests.py b/build/tests.py old mode 100644 new mode 100755 index 3828fe4..b8d1dd0 --- a/build/tests.py +++ b/build/tests.py @@ -1,80 +1,372 @@ +#!/usr/bin/python3 import unittest import json import os import re +import sys +import surok.config +import surok.logger +import surok.discovery +import hashlib +from surok.config import Config -class TestLoadConfig(unittest.TestCase): +class Logger(surok.logger.Logger): + _out='' + _err='' + def _log2err(self,out): + self._err+=out - def test_main_conf(self): - # Load base configurations - surok_conf = '/etc/surok/conf/surok.json' - # Read config file - f = open(surok_conf, 'r') - conf = json.loads(f.read()) - f.close() + def _log2out(self,out): + self._out+=out - self.assertIn('confd', conf) - self.assertTrue(os.path.isdir(conf['confd'])) - self.assertIn('wait_time', conf) - self.assertIn('lock_dir', conf) - self.assertTrue(os.path.isdir(conf['lock_dir'])) + def geterr(self): + return self._err + + def getout(self): + return self._out + + def reset(self): + self._err='' + self._out='' + +class DiscoveryTestingTemplate: + _testing={} + _testing_fqdn_a={ + "test.zzz0.test":['10.0.0.1','10.1.0.1'], + "test.zzz1.test":['10.0.1.1','10.1.1.1'], + "test.zzz2.test":['10.0.2.1','10.1.2.1'], + "test.zzz3.test":['10.0.3.1','10.1.3.1'] + } + _testing_fqdn_srv={} + + def __init__(self): + super().__init__() + self._orig_logger=surok.logger.Logger() + + def do_query_a(self,fqdn): + res=self._testing_fqdn_a.get(fqdn,[]) + if res: + return res + else: + self._orig_logger.error('Testing FQDN '+fqdn+' not found in test A records') + sys.exit(2) + + def do_query_srv(self,fqdn): + res=self._testing_fqdn_srv.get(fqdn,[]) + if res or fqdn.startswith('_tname_e.') or fqdn.find('._udp.'): + return res + else: + self._orig_logger.error('Testing FQDN '+fqdn+' not found in test SRV records') + sys.exit(2) + + def update_data(self): + class_name=self.__class__.__name__ + tgen={ + "name": ["zzz0","zzy0","zzy1","zzz1"], + "host": ["test.zzz0.test","test.zzz1.test","test.zzz2.test","test.zzz3.test"], + "serv": ["tname_aa","tname_ab","tname_ba","tname_bb"], + "ports": [12341,12342,12343,12344], + "servicePorts": [21221,21222,21223,21224] + } + if self._testing.get(class_name,True): + if class_name == 'DiscoveryMarathon': + _tasks=[] + _ports={} + for id in (0,1,2,3): + ports=[]+tgen['ports'] + servicePorts=[]+tgen['servicePorts'] + appId='/'.join(str(tgen['name'][id]+'.xxx.yyy.').split('.')[::-1]) + _ports[appId]=[] + for pid in (0,1,2,3): + ports[pid]+=pid*10 + servicePorts[pid]+=pid*100 + for prot in ['tcp','udp']: + if pid<2 or prot == 'tcp': + _ports[appId].append({'containerPort': 0, + 'hostPort': 0, + 'labels': {}, + 'name': tgen['serv'][pid], + 'protocol': prot, + 'servicePort': servicePorts[pid]}) + + _tasks.append({'appId':appId, + 'host':tgen['host'][id], + 'ports':ports, + 'servicePorts':servicePorts}) + #_tname_a._zzy0.yyy.xxx._tcp.marathon.mesos + self._tasks=_tasks + self._ports=_ports + elif class_name == 'DiscoveryMesos': + for id in (0,1,2,3): + ports=[]+tgen['ports'] + for pid in (0,1,2,3): + ports[pid]+=pid*10 + for prot in ['tcp','udp']: + if pid<2 or prot == 'tcp': + for fqdn in ['_'+tgen['serv'][pid]+'._'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain'), + '_'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain')]: + if not self._testing_fqdn_srv.get(fqdn): + self._testing_fqdn_srv[fqdn]=[] + self._testing_fqdn_srv[fqdn].append({'name':tgen['host'][id],'port':ports[pid]}) + + self._testing[class_name]=False + +class DiscoveryMesos(DiscoveryTestingTemplate,surok.discovery.DiscoveryMesos): + pass + +class DiscoveryMarathon(DiscoveryTestingTemplate,surok.discovery.DiscoveryMarathon): + pass + +class Discovery(surok.discovery.Discovery): + _discoveries={} + def __init__(self): + self._config=Config() + self._logger=Logger() + if not self._discoveries.get('mesos_dns'): + self._discoveries['mesos_dns']=DiscoveryMesos() + if not self._discoveries.get('marathon_api'): + self._discoveries['marathon_api']=DiscoveryMarathon() + +class Test01_Logger(unittest.TestCase): + def test_01_logger_default_level(self): + logger = Logger() + self.assertEqual(logger.get_level(), 'info') + + def test_02_logger_output_levels(self): + message='log message' + tests={ + 'debug':{ + 'assertIn':['ERROR: {}','WARNING: {}','INFO: {}','DEBUG: {}'], + 'assertNotIn':[] + }, + 'info':{ + 'assertIn':['ERROR: {}','WARNING: {}','INFO: {}'], + 'assertNotIn':['DEBUG: {}'] + }, + 'warning':{ + 'assertIn':['ERROR: {}','WARNING: {}'], + 'assertNotIn':['INFO: {}','DEBUG: {}'] + }, + 'error':{ + 'assertIn':['ERROR: {}'], + 'assertNotIn':['WARNING: {}','INFO: {}','DEBUG: {}'] + } + } + logger = Logger() + for value01 in tests.keys(): + logger.reset() + logger.set_level(value01) + logger.error(message) + logger.warning(message) + logger.info(message) + logger.debug(message) + resmessage=logger.geterr()+logger.getout() + for test_name in tests[value01].keys(): + for test_value in tests[value01][test_name]: + with self.subTest(msg='Testing Logger for ...', loglevel=value01): + test_message=test_value.format(message) + eval('self.{}(test_message,resmessage)'.format(test_name)) + +class Test02_LoadConfig(unittest.TestCase): + + def test_01_default_values(self): + config=Config() + with self.subTest(msg='Testing default values for Config.', dump=config.dump()): + self.assertEqual(config.get('confd'), '/etc/surok/conf.d') + self.assertEqual(config.get('default_discovery'), 'mesos_dns') + self.assertEqual(config.get('lock_dir'), '/var/tmp') + self.assertEqual(config.get('loglevel'), 'info') + self.assertEqual(dict(config.get('marathon',{})).get('enabled'), False) + self.assertEqual(dict(config.get('mesos',{})).get('enabled'), False) + self.assertEqual(dict(config.get('memcached',{})).get('enabled'), False) + self.assertEqual(config.get('version'), '0.7') + self.assertEqual(config.get('wait_time'), 20) + + def test_02_main_conf(self): + config=Config('/etc/surok/conf/surok.json') + with self.subTest(msg='Testing load config for Config.', dump=config.dump()): + self.assertEqual(config.hash(), '545c20b322a6ba5fef9c7d2416d80178f26a924b') + + def test_03_apps_conf(self): + tests=[ + { + 'env':{}, + 'self_check.json':'a4e109b9fec696776fd3df091b607e9c1489748c', + 'marathon_check.json':'6be7f26d421d4a0a2e7b089184be0c0e3a50f986' + }, + { + 'env':{'SUROK_DISCOVERY_GROUP':'xxx.yyy'}, + 'self_check.json':'38ab770ff2ba69bf70673288425337ff3c18a807', + 'marathon_check.json':'7b0cb4eab2d8e0f901cc567df28b17279af21baa' + }, + { + 'env':{'MARATHON_APP_ID':'/xxx/yyy/zzz'}, + 'self_check.json':'cbd2a15179649d0e06f98bd64e024481a944d65c', + 'marathon_check.json':'08a382d14285feb1f22b92ba597ecf73d654a2e0' + } + ] + config=Config() + for test in tests: + config.set('env',test['env']) + config.update_apps() + for app in config.apps: + with self.subTest(msg='Testing AppConfig for ...', env=test['env'], conf_name=app.get('conf_name'), dump=app.dump()): + self.assertEqual(test[app.get('conf_name')],app.hash()) + + def test_04_apps_conf(self): + tests={ + 'confd':{ + 'assertEqual': ['/var', '/var/tmp', '/etc/surok/conf.d'], + 'assertNotEqual': [20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True] + }, + 'default_discovery':{ + 'assertEqual':['marathon_api', 'mesos_dns'], + 'assertNotEqual':[20, 'test', None] + }, + 'lock_dir':{ + 'assertEqual':['/var', '/etc/surok/conf.d', '/var/tmp'], + 'assertNotEqual':[20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True] + }, + 'loglevel':{ + 'assertEqual':['error', 'debug', 'info', 'warning'], + 'assertNotEqual':['errrr', 'DEBUG','warn', 'test', 1, None, True] + }, + 'version':{ + 'assertEqual': ['0.7', '0.8'], + 'assertNotEqual': ['0,7', '07', '0.9', 0.7, 0.8, None] + }, + 'wait_time':{ + 'assertEqual': [10, 15, 20], + 'assertNotEqual': ['10', '15', None, True] + } + + } + config=Config() + for name01 in tests.keys(): + oldvalue=config.get(name01) + for test_name in tests[name01].keys(): + for value01 in tests[name01][test_name]: + config.set_config({name01:value01}) + test_value=config.get(name01) + with self.subTest(msg='Testing Config Change for values...', name=name01, value=value01, test_value=test_value): + eval('self.{}(value01, test_value)'.format(test_name)) + config.set(name01,oldvalue) -class TestLogger(unittest.TestCase): - - def test_debug(self): - from surok.logger import Logger - m = Logger() - self.assertIn('DEBUG', m.testing('debug','log message')) - - def test_info(self): - from surok.logger import Logger - m = Logger() - self.assertIn('INFO', m.testing('info','log message')) - - def test_warning(self): - from surok.logger import Logger - m = Logger() - self.assertIn('WARNING', m.testing('warning','log message')) - - def test_error(self): - from surok.logger import Logger - m = Logger() - self.assertIn('ERROR', m.testing('error','log message')) - - -class TestMemcachedDiscovery(unittest.TestCase): - - def test_discovery_memcache(self): - from surok.system import discovery_memcached - from surok.discovery import Discovery - # Load base configurations - surok_conf = '/etc/surok/conf/surok.json' - # Read config file - f = open(surok_conf, 'r') - conf = json.loads(f.read()) - f.close() - d=Discovery(conf) - self.assertEqual(discovery_memcached(conf), []) - - -class TestGetGroup(unittest.TestCase): - - def test_get_group_from_service(self): - from surok.discovery import DiscoveryTemplate - d=DiscoveryTemplate({}) - self.assertEqual('xxx.yyy.zzz',d.get_group({'group':'xxx.yyy.zzz'}, {})) - - def test_get_group_from_env(self): - from surok.discovery import DiscoveryTemplate - d=DiscoveryTemplate({}) - self.assertEqual('xxx.yyy.zzz',d.get_group({}, {'env':{'SUROK_DISCOVERY_GROUP':'xxx.yyy.zzz'}})) - - def test_get_group_from_marathon_id(self): - from surok.discovery import DiscoveryTemplate - d=DiscoveryTemplate({}) - self.assertEqual('xxx.yyy.zzz',d.get_group({}, {'env':{'MARATHON_APP_ID':'/zzz/yyy/xxx/www'}})) +class Test03_Discovery(unittest.TestCase): + def test_01_discovery(self): + tests={ + 'T':{ #mesos_enabled + 'T':{ #marathon_enabled + '0.7':{ #version + 'mesos_dns':{ #default_discovery + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', #app['conf_name'] + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + }, + 'marathon_api':{ + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + }, + 'marathon_api':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + } + } + }, + 'F':{ + '0.7':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + } + } + }, + 'F':{ + 'T':{ + '0.7':{ + 'mesos_dns':{ + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', + 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b', + 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f' + } + } + }, + 'F':{ + '0.7':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + }, + '0.8':{ + 'mesos_dns':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + }, + 'marathon_api':{ + 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f', + 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f' + } + } + } + } + } + config=Config('/etc/surok/conf/surok.json') + config.set('env',{'SUROK_DISCOVERY_GROUP':'xxx.yyy'}) + discovery=Discovery() + for mesos_enabled in tests.keys(): + for marathon_enabled in tests[mesos_enabled].keys(): + for version in tests[mesos_enabled][marathon_enabled].keys(): + for default_discovery in tests[mesos_enabled][marathon_enabled][version].keys(): + config.set_config({'default_discovery':default_discovery, + 'mesos':{'enabled':(mesos_enabled=='T')}, + 'marathon':{'enabled':(marathon_enabled=='T')}, + 'version':version}) + discovery.update_data() + for app in config.apps: + conf_name=app.get('conf_name') + with self.subTest(msg='Testing Discovery for values...', config=config.dump(), conf_name=conf_name): + self.assertEqual(hashlib.sha1(json.dumps(discovery.resolve(app), sort_keys=True).encode()).hexdigest(), + tests[mesos_enabled][marathon_enabled][version][default_discovery][conf_name]) if __name__ == '__main__': unittest.main() diff --git a/conf.d/marathon_check.json b/conf.d/marathon_check.json new file mode 100644 index 0000000..15e784f --- /dev/null +++ b/conf.d/marathon_check.json @@ -0,0 +1,12 @@ +{ + "services": [ + { + "name": "zzy*", + "ports": ["tname_a*"] + } + ], + "template": "templates/selfcheck.jj2", + "dest": "selfcheck", + "reload_cmd": "/bin/echo selfcheck ok", + "discovery": "marathon_api" +} diff --git a/conf.d/self_check.json b/conf.d/self_check.json new file mode 100644 index 0000000..4ce08e7 --- /dev/null +++ b/conf.d/self_check.json @@ -0,0 +1,17 @@ +{ + "services": [ + {"name":"zzy0", + "ports":["tname_aa","tname_ab","tname_ba","tname_bb","tname_d"] + }, + {"name":"zzy1", + "ports":["tname_aa","tname_ab","tname_ba","tname_bb"] + }, + {"name":"zzz0", + "ports":["tname_aa","tname_bb"] + }, + {"name":"zzz1"} + ], + "template": "templates/selfcheck.jj2", + "dest": "selfcheck", + "reload_cmd": "/bin/echo selfcheck ok" +} diff --git a/conf.d/selfcheck.json b/conf.d/selfcheck.json deleted file mode 100644 index 56cec93..0000000 --- a/conf.d/selfcheck.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "services": [ - { - "name": "", - "group": "mesos" - } - ], - "conf_name": "selfcheck", - "template": "templates/selfcheck.jj2", - "dest": "selfcheck", - "reload_cmd": "/bin/echo selfcheck ok" -} diff --git a/conf/surok.json b/conf/surok.json index e1c58f2..20bd0fd 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,16 +1,13 @@ { + "version":"0.8", "marathon": { "enabled": false, "restart": false, "force": true, "host": "http://marathon.mesos:8080" }, - "consul": { - "enabled": false, - "domain": "service.dc1.consul" - }, "mesos":{ - "enabled": true, + "enabled": false, "domain": "marathon.mesos" }, "default_discovery": "mesos_dns", @@ -18,7 +15,6 @@ "wait_time": 20, "lock_dir": "/var/tmp", "loglevel": "info", - "container": false, "memcached": { "enabled": false, "discovery": { diff --git a/debian/install b/debian/install index 74eab76..d00a88b 100644 --- a/debian/install +++ b/debian/install @@ -1,7 +1,11 @@ conf/surok.json etc/surok/conf +conf.d/self_check.json etc/surok/conf.d +conf.d/marathon_check.json etc/surok/conf.d +templates/selfcheck.jj2 etc/surok/templates surok/templates.py opt/surok/surok surok/system.py opt/surok/surok surok/logger.py opt/surok/surok surok/__init__.py opt/surok/surok surok/discovery.py opt/surok/surok +surok/config.py opt/surok/surok surok.py opt/surok diff --git a/docs/README.md b/docs/README.md index 82774d8..a5815d2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,6 +31,7 @@ Service discovery for Apache Mesos. * Denis Zheleztsov * Denis Ryabyy +* Evgeniy Vasilev ## LICENSE diff --git a/docs/ru/surok.configuration.md b/docs/ru/surok.configuration.md index 213ae15..754bc5d 100644 --- a/docs/ru/surok.configuration.md +++ b/docs/ru/surok.configuration.md @@ -1,30 +1,89 @@ -# Конфигурация Surok (0.7.x) - -**/etc/surok/conf/surok.json** -Разберем конфигурационный файл по опциям +# Конфигурация Surok (0.8.x) +**/etc/surok/conf/surok.json** Разберем конфигурационный файл по опциям ``` { + "version": "0.8" "marathon": { - "force": true, - "host": "marathon.mesos:8080", - "enabled": true + "enabled": false, + "restart": false, + "force": true, + "host": "http://marathon.mesos:8080" }, + "consul": { + "enabled": false, + "domain": "service.dc1.consul" + }, + "mesos":{ + "enabled": true, + "domain": "marathon.mesos" + }, + "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", - "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", - "loglevel": "info|debug" - "container": true|false + "loglevel": "info", + "memcached": { + "enabled": false, + "discovery": { + "enabled": false, + "service": "memcached", + "group": "system" + }, + "hosts": ["localhost:11211"] + } } ``` +## Опции файла конфигурации +* **version** - *string. Не обязательный. По умолчанию "0.7".* +Версия файлов конфигурации, шаблонов. На текущий момент может принимать значения "0.7" или "0.8". + * значение "0.7" - файлы конфигурации версии 0.7.х и более ранних + * значение "0.8" - файлы конфигурации версии 0.8 + +##### версия 0.8 +* **marathon**, **mesos**, **consul**, **memcached** - *dict/hash. Не обязательный. По умолчанию '{"enable":false}'.* +Системы с которыми работает сурок. Если система выключена, то параметры системы и их наличие уже не важны. + * **enable** - *boolean. Не обязательный. По умолчанию false.* + Доступность системы для использования. + + специфичные параметры: + * для Marathon API "marathon" + * **force** - *boolean. Не обязательный. По умолчанию true.* + Рестарт контейнера с force или нет. + * **restart** - *boolean. Не обязательный. По умолчанию false.* + Вкл/выкл. рестарта контейнера + * **host** - *string. Не обязательный. По умолчанию "http://marathon.mesos:8080".* + Адрес Marathon. + * для Consul "consul" + * **domain** - *string. Обязательный.* + Приватный домен Consul + * для mesos DNS "mesos" + * **domain** - *string. Не обязательный. По умолчанию "marathon.mesos".* + Приватный домен Mesos DNS + * для Memcached "memcached" + * **hosts** - + * **discovery** - + * **enabled** - + * **service** - + * **group** - +* **default_discovery** - *string. Не обязательный. По умолчанию "mesos_dns".* + + Может принимать значения: + * "mesos_dns" - Mesos DNS + * "marathon_api"- Marathon API + * "consul_dns" - Consul +* **confd** - *strig. Обязательный.* + Абсолютный путь до директории с конфигурационными файлами приложений. +* **wait_time** - *int. Обязательный.* + Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов. +* **lock_dir** - *string. Обязательный.* + Абсолютный путь до директории с lock-конфигурациями. +* **loglevel** - *string. Не обязательный. По умолчанию "info".* + Уровень логирования. Может принимать значения: "debug", "info", "warning", "error" + +##### версия 0.7 и более ранние +Особенности для файла конфигурации +* **marathon** + * **enabled** - boolean. Вкл/выкл. рестарта контейнера. В версии 0.8 переименована в "restart". +* **domain** - string. Приватный домен Mesos DNS. В версии 0.8 перемещен в dict "mesos". + Обнаружение Mesos DNS включено всегда. -* marathon(v0.7) - hash. В текущей версии отвечает за перезапуск контейнера. Обнаружение сервисов через Marathon пока недоступно. - 1. force - boolean. Рестарт контейнера с force или нет. - 2. host - string. Адрес Marathon. - 3. enabled - boolean. Вкл/выкл. -* confd - strig. Абсолютный путь до директории с конфигурационными файлами приложений. -* domain - string. Домен, который обслуживает mesos-dns. -* wait_time - int. Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов. -* lock_dir - string. Абсолютный путь до директории с lock-конфигурациями. -* loglevel - string. Уровень логирования. -* container(v0.6) - boolean. Определяем внутри или нет контейнера запущен сурок. Меняется логика работы. diff --git a/docs/ru/templates.md b/docs/ru/templates.md index 4b98f32..c7609a5 100644 --- a/docs/ru/templates.md +++ b/docs/ru/templates.md @@ -4,6 +4,127 @@ ## Словарь my в шаблоне +### Версия 0.8 +Surok заполняет словарь my и передает его в шаблон. +``` +{ + "services": { + "asterisk": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1", + "11.0.0.1" + ], + "tcp": { + "rpc":31200, + "web":31201, + "sip":32000 + }, + "udp": { + "sip":31201 + } + }, + { + "name": "nginx.testing-kl123-s1.marathon.mesos.", + "ip": [ + "10.0.0.2", + "11.0.0.2" + ], + "tcp": { + "rpc":31210, + "web":31211, + "sip":32010 + }, + "udp": { + "sip":31211 + } + } + ], + "email": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1" + ], + "tcp": { + "smtp":31200, + "pop":31201 + } + } + ], + "anyport": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1" + ], + "tcp": [ + 31200, + 31201 + ] + } + ] + "env": { + "HOME": "/var/lib/nginx" + } +} +``` + +## Пример реального шаблона + +``` +upstream matrix-http { + hash $remote_addr; +{% for server in my['services']['matrix'] %} + server {{server['name']}}:{{server['tcp']['http']}} max_fails=3; +{% endfor %} +} + +upstream riot-http { + hash $remote_addr; +{% for server in my['services']['riot'] %} + server {{server['name']}}:{{server['tcp'][0]}} max_fails=3; +{% endfor %} +} + +server { + listen 10.15.56.157:80; + server_name matrix.example.com; + + client_max_body_size 10m; + + location / { + proxy_pass http://riot-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /_matrix/ { + proxy_pass http://matrix-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + +} +``` +Так для upstream matrix-http используются именованные порты, а для riot-http – нет. + +## Проверки в шаблоне + +Переменная _my['env']_ является классом python _os.environ_, что позваоляет нам строить различные проверки, например: + +``` +{% if my['env'].get('DB_HOST') -%} +host = '{{my['env']['DB_HOST']}}' +{% else -%} +host = 'localhost' +{% endif -%} +``` + +### Версия 0.7 Surok заполняет словарь my и передает его в шаблон. ``` { @@ -29,13 +150,13 @@ Surok заполняет словарь my и передает его в шаб } ], "service-with-defined-ports": { - "web": [ + "name-of-port0": [ { "name": "f.q.d.n", "port": 12341 } ], - "rpc": [ + "name-of-port2": [ { "name": "f.q.d.n", "port": 12342 @@ -58,34 +179,34 @@ upstream matrix-http { server {{server['name']}}:{{server['port']}} max_fails=3; {% endfor %} } - + upstream riot-http { hash $remote_addr; {% for server in my['services']['riot'] %} server {{server['name']}}:{{server['port']}} max_fails=3; {% endfor %} } - + server { listen 10.15.56.157:80; server_name matrix.example.com; - + client_max_body_size 10m; - + location / { proxy_pass http://riot-http; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - + location /_matrix/ { proxy_pass http://matrix-http; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - + } ``` Так для upstream matrix-http используются именованные порты, а для riot-http – нет. diff --git a/surok.py b/surok.py index 4429e19..e9835f2 100755 --- a/surok.py +++ b/surok.py @@ -10,45 +10,17 @@ from surok.templates import gen from surok.discovery import Discovery from surok.system import reload_conf from surok.logger import Logger +from surok.config import Config logger=Logger() -# Load base configurations -surok_conf = '/etc/surok/conf/surok.json' # Command line arguments parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', help='surok.json path') args = parser.parse_args() -if args.config: - surok_conf = args.config -# Read config file -f = open(surok_conf, 'r') -conf = json.loads(f.read()) -f.close() - - -# Get app configurations -# Return list of patches to app discovery configuration -def get_configs(): - confs = [f for f in listdir(conf['confd']) if isfile( - join(conf['confd'], f))] - return sorted(confs) - - -# Get Surok App configuration -# Read app conf from file and return dict -def load_app_conf(app): - # Load OS environment to app_conf - f = open(conf['confd'] + '/' + app) - c = json.loads(f.read()) - f.close() - - c['env'] = os.environ - - return c - -logger.set_level(conf.get('loglevel','info')) +# Load base configurations +config=Config(args.config if args.config else '/etc/surok/conf/surok.json') # Main loop ########### @@ -56,29 +28,21 @@ logger.set_level(conf.get('loglevel','info')) discovery=Discovery() while 1: - confs = get_configs() - # Update config from discovery object - discovery.set_config(conf) - - # Update discovery data discovery.update_data() + for app in config.apps: - for app in confs: - app_conf = load_app_conf(app) - - # Resolve services - app_hosts = discovery.resolve(app_conf) + app_hosts = discovery.resolve(app) # Populate my dictionary my = {"services": app_hosts, - "conf_name": app_conf['conf_name']} + "conf_name": app['conf_name']} logger.debug('my=',my) # Generate config from template - service_conf = gen(my, app_conf['template']) + service_conf = gen(my, app['template']) - reload_conf(service_conf, app_conf, conf, app_hosts) + reload_conf(service_conf, app, config, app_hosts) - sleep(conf['wait_time']) + sleep(config['wait_time']) diff --git a/surok.spec b/surok.spec index 82289d1..33c1a53 100644 --- a/surok.spec +++ b/surok.spec @@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates} install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf -install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/requriments.txt %{buildroot}/opt/surok @@ -59,7 +61,11 @@ cd /opt/surok && pip3 install -r requriments.txt /opt/surok/surok/templates.py /opt/surok/surok/templates.pyc /opt/surok/surok/templates.pyo -/etc/surok/conf.d/selfcheck.json +/opt/surok/surok/config.py +/opt/surok/surok/config.pyc +/opt/surok/surok/config.pyo +/etc/surok/conf.d/self_check.json +/etc/surok/conf.d/marathon_check.json /etc/surok/templates/selfcheck.jj2 /opt/surok/requriments.txt %defattr(-,root,root,-) diff --git a/surok/config.py b/surok/config.py new file mode 100644 index 0000000..4d58742 --- /dev/null +++ b/surok/config.py @@ -0,0 +1,410 @@ +# Public names +__all__ = ['Config', 'AppConfig'] + +import hashlib +import importlib +import json +import os +from .logger import * + +# Logger link + +# Config singleton link +_config_singleton = None + +''' +Test values +================================================== +key - key +value - value of key +type_value - type of value +type_par - additional parameters for test + +''' + + + +''' +Public Config object +================================================== +.set_config(conf_data) - set config data + Use: conf_data(str type) - path of json config file + conf_data(dict type) - dict with config +.set(key,value) - set config key +.get(key) - get config key +.update_apps() - update apps config data +.apps - Apps object. List of AppConfig oblects +''' + +class _ConfigTemplate(dict): + _conf={} + def _init_conf(self,params): + conf={} + for k in params.keys(): + if params[k].get('params'): + conf[k]=self._init_conf(params[k].get('params')) + else: + if params[k].get('value') is not None: + conf[k]=params[k].get('value') + return conf + + def __init__(self, *conf_data): + self._logger = Logger() + if not self._conf: + self._conf = self._init_conf(self._params) + for c in conf_data: + self.set_config(c) + + def _set_conf_params(self,oldconf,testconf,params): + conf = oldconf if oldconf else {} + for key in testconf.keys(): + resvalue=None + param=params.get(key) + oldvalue=conf.get(key) + testvalue=testconf.get(key) + if param is None: + self._logger.error('Parameter "', key, '" value "', testvalue, '" type is "', type(testvalue).__name__, '" not found') + else: + type_param=param.get('type') + resvalue=[] + if type(testvalue).__name__ != 'list': + testvalue=[testvalue] + for testitem in testvalue: + if self._test_value(key, testitem, param): + if 'dict' in type_param: + if param.get('params'): + res=self._set_conf_params(oldvalue,testitem,param.get('params')) + if res is not None: + resvalue.append(res) + else: + resvalue.append(testitem) + if 'list' not in type_param: + resvalue=list([None]+resvalue).pop() + if resvalue is not None and 'do' in type_param: + if not self._do_type_set(key, resvalue, param): + self._logger.warning('Parameter "', key, '" current "', resvalue, '" type is "', type(resvalue).__name__, '" testing failed') + resvalue = None + if resvalue is not None: + conf[key]=resvalue + return conf + + def _test_value(self, key, value, param): + type_param=param.get('type') + type_value=[x for x in type_param if x in ['str', 'int', 'bool', 'dict']] + if type_value: + if type(value).__name__ not in type_value: + self._logger.error('Parameter "', key, '" must be ', type_value,' types, current "', value, '" (',type(value).__name__,')') + return False + if 'value' in type_param: + if value not in param.get('values',[]): + self._logger.error('Value "', value, '" of key "', key, '" unknown') + return False + if 'dir' in type_param: + if not os.path.isdir(value): + self._logger.error('Path "{}" not present'.format(value)) + return False + elif 'file' in type_param: + if not os.path.isfile(value): + self._logger.error('File "{}" not present'.format(value)) + return False + return True + else: + self._logger.error('Type for testing "{}" unknown'.format(type_value)) + return False + + def set_config(self, conf_data): + conf = {} + if type(conf_data).__name__ == 'str': + try: + self._logger.debug('Open file ', conf_data) + f = open(conf_data, 'r') + json_data=f.read() + f.close() + conf = json.loads(json_data) + except OSError as err: + self._logger.error("OS error: {0}".format(err)) + pass + except ValueError as err: + self._logger.error('JSON format error: {0}'.format(err)) + pass + except: + self._logger.error('Load main config file failed') + pass + elif type(conf).__name__ == 'dict': + conf = conf_data + else: + return False + self._conf=self._set_conf_params(self._conf,conf,self._params) + self._logger.debug('Conf=', self._conf) + + def keys(self): + return self._conf.keys() + + def dump(self): + return json.dumps(self._conf, sort_keys=True, indent=2) + + def _do_type_set(self, key, value, params): + self._logger.error('_do_type_set handler is not defined') + return False + + def hash(self): + return hashlib.sha1(json.dumps(self._conf, sort_keys=True).encode()).hexdigest() + + def set(self, key, value): + self._conf[key]=value + + def __setitem__(self, key, value): + self.set(key, value) + + def get(self, key, default=None): + return self._conf.get(key, default) + + def __getitem__(self, key): + return self.get(key) + + def __contains__(self, item): + return bool(item in self._conf) + + def __len__(self): + return self._conf.__len__() + + def __str__(self): + return self._conf.__str__() + + def __repr__(self): + return self._conf.__repr__() + + +class Config(_ConfigTemplate): + _params = { + 'marathon': { + 'params': { + 'force': { + 'value': True, + 'type': ['bool'] + }, + 'host': { + 'value': 'http://marathon.mesos:8080', + 'type': ['str'] + }, + 'enabled': { + 'value': False, + 'type': ['bool'] + }, + 'restart': { + 'value': False, + 'type': ['bool'] + } + }, + 'type':['dict'] + }, + 'mesos': { + 'params': { + "domain": { + 'value': "marathon.mesos", + 'type': ['str'] + }, + "enabled": { + 'value': False, + 'type': ['bool'] + } + }, + 'type':['dict'] + }, + 'memcached': { + 'params': { + 'enabled': { + 'value': False, + 'type': ['bool'] + }, + 'discovery': { + 'params': { + 'enabled': { + 'value': False, + 'type': ['bool'] + }, + 'service': { + 'value': 'memcached', + 'type': ['str'] + }, + 'group': { + 'value': 'system', + 'type': ['str'] + } + }, + 'type':['dict'] + }, + 'hosts': { + 'value': ['localhost:11211'], + 'type': ['list','str'] + } + }, + 'type':['dict'] + }, + 'version': { + 'value': '0.7', + 'type': ['str','value'], + 'values': ['0.7', '0.8'] + }, + 'confd': { + 'value': '/etc/surok/conf.d', + 'type': ['str','dir'] + }, + 'wait_time': { + 'value': 20, + 'type': ['int'] + }, + 'lock_dir': { + 'value': '/var/tmp', + 'type': ['str','dir'] + }, + 'default_discovery': { + 'value': 'mesos_dns', + 'type': ['str', 'value'], + 'values': ['mesos_dns', 'marathon_api'] + }, + 'loglevel': { + 'value': 'info', + 'type': ['str','do'], + 'do':'set_loglevel' + } + } + + def __new__(cls, *args): + global _config_singleton + if _config_singleton is None: + _config_singleton = super(Config, cls).__new__(cls) + return _config_singleton + + def __init__(self, *conf_data): + super().__init__( *conf_data) + self.apps = _Apps() + + def set_config(self, conf_data): + super().set_config(conf_data) + if self.get('version') == '0.7': + domain = self.get('domain') + if domain is not None and self.get('mesos') is None: + self.set('mesos', {'domain': domain, 'enabled': True}) + + def _do_type_set(self, key, value, param): + if param.get('do') == 'set_loglevel': + if self._logger.set_level(value): + return True + return False + + def update_apps(self): + self.apps.reset() + for app in sorted([os.path.join(self.get('confd'), f) for f in os.listdir(self.get('confd')) if os.path.isfile(os.path.join(self.get('confd'), f))]): + self.apps.set(AppConfig(app)) + +''' +Private Apps object +================================================== +.get(app) - get _AppConfig object +.set(app) - set _AppConfig object +''' + + +class _Apps: + _apps = {} + _items = [] + + def get(self, key): + return self._apps.get(key) + + def set(self, app): + self._apps[app.get('conf_name')] = app + + def reset(self): + keys=[]+list(self.keys()) + for k in keys: + del self._apps[k] + + def __getitem__(self, key): + return self.get(key) + + def __contains__(self, item): + return bool(item in self._apps) + + def __iter__(self): + self._items = sorted(self.keys()) + return self + + def __next__(self): + if self._items: + return self.get(self._items.pop(0)) + raise StopIteration + + def keys(self): + return self._apps.keys() + +''' +Public AppConfig object +================================================== +.set_config(conf_data) - set config data + Use: conf_data(str type) - path of json config file + conf_data(dict type) - dict with config +.set(key,value) - set config key +.get(key) - get config key +''' + + +class AppConfig(_ConfigTemplate): + _params = { + 'conf_name': { + 'type': ['str'] + }, + 'services': { + 'params': { + 'name': { + 'type': ['str'] + }, + 'ports': { + 'type': ['list','str'] + }, + 'discovery': { + 'type': ['str'] + }, + 'group': { + 'type': ['str'] + } + }, + 'type':['list','dict'] + }, + 'template': { + 'type': ['str','file'] + }, + 'dest': { + 'type': ['str'] + }, + 'reload_cmd': { + 'type': ['str'] + }, + 'discovery': { + 'type': ['str'] + }, + 'group': { + 'type': ['str'] + } + } + + def __init__(self, *conf_data): + self._config = Config() + super().__init__(*conf_data) + + def set_config(self, conf_data): + super().set_config(conf_data) + self._conf.setdefault('discovery', self._config.get('default_discovery')) + self._conf.setdefault('group', self._get_default_group()) + if type(conf_data).__name__ == 'str': + self._conf.setdefault('conf_name', os.path.basename(conf_data)) + + def _get_default_group(self): + env=self._config.get('env',dict(os.environ)) + # Check environment variable + if env.get('SUROK_DISCOVERY_GROUP'): + return env['SUROK_DISCOVERY_GROUP'] + # Check marathon environment variable + elif env.get('MARATHON_APP_ID'): + return ".".join(env['MARATHON_APP_ID'].split('/')[-2:0:-1]) diff --git a/surok/discovery.py b/surok/discovery.py index 429e51b..8018b7f 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -1,282 +1,217 @@ +#Public names +__all__=['Discovery','DiscoveryMesos','DiscoveryMarathon'] import dns.resolver import dns.query -from dns.exception import DNSException -from .logger import Logger +import os import sys import requests +from dns.exception import DNSException +from .config import * +from .logger import * -# Default config for Discovery class -_config={ - 'default_discovery':'mesos_dns' # Default discovery system -} - -# Discoveries objects -_discoveries={} - -#Logger -logger=Logger() +# Discovery object +_discovery_singleton=None class DiscoveryTemplate: - # Default config values for discovery template - _config={} - _defconfig={'enabled':False} - - def __init__(self,conf): - for key in self._defconfig.keys(): - if key not in self._config.keys(): - self._config[key]=self._defconfig[key] - self.set_config(conf) - - def set_config(self,conf): - pass + def __init__(self): + self._config=Config() + self._logger=Logger() def enabled(self): - return self._config['enabled'] + return self._config[self._config_section].get('enabled',False) def update_data(self): pass - def get_group(self,service, app): - # Check group in app conf - if 'group' in service: - return service['group'] + # Do DNS queries + # Return array: + # ["10.10.10.1", "10.10.10.2"] + def do_query_a(self,fqdn): + servers = [] + try: + resolver = dns.resolver.Resolver() + for a_rdata in resolver.query(fqdn, 'A'): + servers.append(a_rdata.address) + except DNSException as e: + self._logger.error("Could not resolve ",fqdn) + return servers - # Check environment variable - elif app['env'].get('SUROK_DISCOVERY_GROUP'): - return app['env']['SUROK_DISCOVERY_GROUP'] - - # Check marathon environment variable - elif app['env'].get('MARATHON_APP_ID'): - return ".".join(app['env']['MARATHON_APP_ID'].split('/')[-2:0:-1]) - - else: - logger.error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID') - logger.error('Not in Mesos launch?') - sys.exit(2) + # Do DNS queries + # Return array: + # [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}] + def do_query_srv(self,fqdn): + servers = [] + try: + resolver = dns.resolver.Resolver() + resolver.lifetime = 1 + resolver.timeout = 1 + query = resolver.query(fqdn, 'SRV') + for rdata in query: + info = str(rdata).split() + servers.append({'name': info[3][:-1], 'port': info[2]}) + except DNSException as e: + self._logger.warning("Could not resolve ",fqdn) + return servers class Discovery: - def __init__(self,*conf): - for __conf in conf: - self.set_config(__conf) + _discoveries={} + def __new__(cls): + global _discovery_singleton + if _discovery_singleton is None: + _discovery_singleton=super(Discovery, cls).__new__(cls) + return _discovery_singleton - def set_config(self,conf): - global _discoveries - #Get discoveries objects - if not _discoveries.get('mesos_dns'): - _discoveries['mesos_dns']=DiscoveryMesos(conf) - else: - _discoveries['mesos_dns'].set_config(conf) + def __init__(self): + self._config=Config() + self._logger=Logger() + if not self._discoveries.get('mesos_dns'): + self._discoveries['mesos_dns']=DiscoveryMesos() + if not self._discoveries.get('marathon_api'): + self._discoveries['marathon_api']=DiscoveryMarathon() - if not _discoveries.get('marathon_api'): - _discoveries['marathon_api']=DiscoveryMarathon(conf) - else: - _discoveries['marathon_api'].set_config(conf) - - if not _discoveries.get('consul_dns'): - _discoveries['consul_dns']=DiscoveryConsul(conf) - else: - _discoveries['consul_dns'].set_config(conf) - - global _config - if conf.get('default_discovery'): - discovery=conf.get('default_discovery') - if discovery in list(_discoveries.keys()): - _config['default_discovery']=discovery - else: - logger.error('Default discovery "'+discovery+'" is not present') - logger.debug('Conf=',conf) + def keys(self): + return self._discoveries.keys() def resolve(self,app): - __discovery=_config.get('default_discovery') - if app.get('discovery'): - discovery=app.get('discovery') - if discovery in list(_discoveries.keys()): - __discovery=discovery - else: - logger.warning('Discovery "'+discovery+'" is not present') - logger.debug('App=',app) - return {} - if _discoveries[__discovery].enabled(): - return _discoveries[__discovery].resolve(app) + discovery=app.get('discovery',self._config.get('default_discovery')) + if discovery not in self.keys(): + self._logger.warning('Discovery "',discovery,'" is not present') + return {} + if self._discoveries[discovery].enabled(): + return self.compatible(self._discoveries[discovery].resolve(app)) else: - logger.error('Discovery "'+__discovery+'" is disabled') + self._logger.error('Discovery "',discovery,'" is disabled') return {} def update_data(self): - global _discoveries - for d in list(_discoveries.keys()): - if _discoveries[d].enabled(): - _discoveries[d].update_data() + self._config.update_apps() + for d in self.keys(): + if self._discoveries[d].enabled(): + self._discoveries[d].update_data() + + def compatible(self,hosts): + compatible_hosts={} + if self._config.get('version') == '0.7': + for service in hosts.keys(): + for host in hosts[service]: + ports=host.get('tcp',[]) + if type(ports).__name__ == 'list': + compatible_hosts[service]=[] + for port in ports: + compatible_hosts[service].append({'name':host['name'], + 'ip':host['ip'], + 'port':str(port)}) + else: + compatible_hosts[service]={} + for port in ports.keys(): + compatible_host=compatible_hosts[service].setdefault(port,[]) + compatible_host.append({'name':host['name'], + 'ip':host['ip'], + 'port':ports[port]}) + + return compatible_hosts + return hosts class DiscoveryMesos(DiscoveryTemplate): - _config={ - 'domain':'marathon.mesos' # Default domain - } - - def set_config(self,conf): - # For old version config - if conf.get('domain'): - self._config['domain']=conf.get('domain') - self._config['enabled']=True - # For current version config - if conf.get('mesos'): - _conf=conf['mesos'] - for p in ['domain','enabled']: - if _conf.get(p): - self._config[p]=_conf.get(p) - + _config_section='mesos' def resolve(self,app): hosts = {} - services = app['services'] - domain = self._config['domain'] + services = app.get('services') + domain = self._config[self._config_section].get('domain') for service in services: - group = self.get_group(service, app) + group = service.get('group',app.get('group')) + if group is None: + self._logger.error('Group for service "{}" of config "{}" not found'.format(service['name'],app.get('conf_name'))) + continue ports = service.get('ports') name = service['name'] hosts[name] = {} serv = hosts[name] - if ports is not None: - hosts[name] = {} - serv = hosts[name] - for prot in ['tcp','udp']: + self._logger.debug('group=',group,' ports=',ports,' name=',name,' serv=',serv) + for prot in ['tcp','udp']: + if ports is not None: for port_name in ports: - for d in do_query('_'+port_name+'._'+name+'.'+group+'._'+prot+'.'+domain): + for d in self.do_query_srv('_'+port_name+'._'+name+'.'+group+'._'+prot+'.'+domain): hostname=d['name'] - if serv.get(hostname) is None: - serv[hostname]={"name":hostname,"ip":d['ip']} - if serv[hostname].get(prot) is None: - serv[hostname][prot]={} - serv[hostname][prot][port_name]=d['port'] - hosts[name]=list(hosts[name].values()) - else: - hosts[name]=do_query('_'+name+'.'+group+'._tcp.'+domain) - + serv.setdefault(hostname,{'name':hostname, + 'ip':self.do_query_a(hostname)}) + serv[hostname].setdefault(prot,{}) + serv[hostname][prot][port_name]=d['port'] + else: + for d in self.do_query_srv('_'+name+'.'+group+'._'+prot+'.'+domain): + hostname=d['name'] + if serv.get(hostname) is None: + serv[hostname]={'name':hostname, + 'ip':self.do_query_a(hostname)} + if serv[hostname].get(prot) is None: + serv[hostname][prot]=[] + serv[hostname][prot].extend([d['port']]) + hosts[name]=list(serv.values()) return hosts class DiscoveryMarathon(DiscoveryTemplate): - _config={ - 'host':'http://marathon.mesos:8080', - 'force':True - } - __tasks = [] - __ports = {} - def set_config(self,conf): - # For current version config - if conf.get('marathon'): - _conf=conf['marathon'] - for p in ['host','enabled','force']: - if _conf.get(p): - self._config[p]=_conf.get(p) + _config_section='marathon' + _tasks = [] + _ports = {} def update_data(self): + hostname=self._config[self._config_section].get('host') try: - apps = requests.get(self._config['host']+'/v2/apps').json()['apps'] ports = {} - for app in apps: + for app in requests.get(hostname+'/v2/apps').json()['apps']: ports[app['id']] = {} - if app.get('container') is not None and app['container']['type'] == 'DOCKER': + if app.get('container') is not None and app['container'].get('type') == 'DOCKER': ports[app['id']] = app['container']['docker'].get('portMappings',[]) - self.__ports=ports + self._ports=ports except: - logger.warning('Apps ('+self._config['host']+'/v2/apps) request from Marathon API is failed') + self._logger.warning('Apps (',hostname,'/v2/apps) request from Marathon API is failed') pass try: - self.__tasks = requests.get(self._config['host']+'/v2/tasks').json()['tasks'] + self._tasks = requests.get(hostname + '/v2/tasks').json()['tasks'] except: - logger.warning('Tasks ('+self._config['host']+'/v2/tasks) request from Marathon API is failed') + self._logger.warning('Tasks (',hostname,'/v2/tasks) request from Marathon API is failed') pass + def _test_mask(self, mask, value): + return (mask.endswith('*') and value.startswith(mask[:-1])) or mask == value + def resolve(self, app): hosts={} - serv_conf = app['services'] - if not serv_conf: - serv_conf = [{'name':'*','ports':['*']}] - for serv in serv_conf: - # Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format - group = '/'.join(['']+self.get_group(serv, app).split('.')[::-1]+['']) - mask = group+serv['name'] - for task in self.__tasks: - if (mask.endswith('*') and task['appId'].startswith(mask[:-1])) or task['appId'] == mask: - name='.'.join(task['appId'][len(group):].split('/')[::-1]) - if 'ports' in serv: - hosts[name]={} - for port in self.__ports[task['appId']]: - for pp in serv['ports']: - if (pp.endswith('*') and port['name'].startswith(pp[:-1])) or port['name'] == pp: - if hosts[name].get(task['host']) is None: - hosts[name][task['host']]={'name':task['host'], - 'ip':do_query_a(task['host'])} - if hosts[name][task['host']].get(port['protocol']) is None: - hosts[name][task['host']][port['protocol']]={} - hosts[name][task['host']][port['protocol']][port['name']]=task['ports'][task['servicePorts'].index(port['servicePort'])] - hosts[name]=list(hosts[name].values()) - else: - hosts[name]=[] - for port in self.__ports[task['appId']]: - hosts[name].append({'name':task['host'], - 'port':task['ports'][task['servicePorts'].index(port['servicePort'])], - 'ip':do_query_a(task['host'])}) - - return hosts - - -class DiscoveryConsul(DiscoveryTemplate): - _config={ - 'enabled':False, - 'domain':None - } - def set_config(self,conf): - # For current version config - if conf.get('consul'): - _conf=conf['consul'] - for p in ['domain','enabled']: - if _conf.get(p): - self._config[p]=_conf.get(p) - - def resolve(self,app): - hosts = {} - services = app['services'] - domain = self._config['domain'] + services = app.get('services') + if not services: + services = [{'name':'*','ports':['*']}] for service in services: - name = service['name'] - hosts[name]=do_query('_'+name+'._tcp.'+domain) + # Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format + group=service.get('group',app.get('group')) + if group is None: + self._logger.error('Group for service "{}" of config "{}" not found'.format(service['name'],app.get('conf_name'))) + continue + group = '/'+'/'.join(group.split('.')[::-1])+'/' + service_mask = group + service['name'] + for task in self._tasks: + if self._test_mask(service_mask,task['appId']): + name='.'.join(task['appId'][len(group):].split('/')[::-1]) + hosts[name]={} + serv = hosts[name] + hostname=task['host'] + for task_port in self._ports[task['appId']]: + prot=task_port['protocol'] + port_name=task_port['name'] + port=task['ports'][task['servicePorts'].index(task_port['servicePort'])] + if 'ports' in service: + for port_mask in service['ports']: + if self._test_mask(port_mask,port_name): + serv.setdefault(hostname,{'name':hostname, + 'ip':self.do_query_a(hostname)}) + serv[hostname].setdefault(prot,{}) + serv[hostname][prot][port_name]=port + else: + serv.setdefault(hostname,{'name':hostname, + 'ip':self.do_query_a(hostname)}) + serv[hostname].setdefault(prot,[]) + serv[hostname][prot].extend([port]) + hosts[name]=list(serv.values()) return hosts - - -# Do DNS queries -# Return array: -# ["10.10.10.1", "10.10.10.2"] -def do_query_a(fqdn): - servers = [] - try: - resolver = dns.resolver.Resolver() - for a_rdata in resolver.query(fqdn, 'A'): - servers.append(a_rdata.address) - except DNSException as e: - logger.error("Could not resolve "+fqdn) - - return servers - - -# Do DNS queries -# Return array: -# [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}] -def do_query(fqdn): - servers = [] - try: - resolver = dns.resolver.Resolver() - resolver.lifetime = 1 - resolver.timeout = 1 - query = resolver.query(fqdn, 'SRV') - for rdata in query: - info = str(rdata).split() - name = info[3][:-1] - port = info[2] - servers.append({'name': name, 'port': port, 'ip': do_query_a(name)}) - except DNSException as e: - logger.error("Could not resolve " + fqdn) - - return servers diff --git a/surok/logger.py b/surok/logger.py index 7cee349..eec8efb 100644 --- a/surok/logger.py +++ b/surok/logger.py @@ -1,51 +1,86 @@ +# Public names +__all__ = ['Logger'] + import sys import json -from time import time -_loglevel='info' -msg_level={'debug':'DEBUG', - 'info':'INFO', - 'warning':'WARNING', - 'error':'ERROR'} +import time + +# Logger singleton link +_logger_singleton = None + +''' +Public Logger oblect +================================================== +.set_level(level) - set level messages + level - values 'debug', 'info', 'warning', 'error' + * error - write error message + * warning - write warning and error message + * info - write info, warning and error message + * debug - write all message +.get_level() - get level messages +.error(str) - write error message +.warning(str) - write warning message +.info(str) - write info message +.debug(str)- write error message +''' + class Logger: - def __init__(self,*args): + _loglevel = 'info' + _msg_level = { + 'debug': 'DEBUG', + 'info': 'INFO', + 'warning': 'WARNING', + 'error': 'ERROR' + } + + def __new__(cls, *args): + global _logger_singleton + if _logger_singleton is None: + _logger_singleton = super(Logger, cls).__new__(cls) + return _logger_singleton + + def __init__(self, *args): if args: self.set_level(args[0]) - def set_level(self,level): - if level in ['debug','info','warning','error']: - global _loglevel - _loglevel=level + def set_level(self, level): + if level in ['debug', 'info', 'warning', 'error']: + self._loglevel = level + return True + else: + self.warning('Log level "', level, '" not valid') + return False def get_level(self): - return _loglevel + return self._loglevel - def __make_message(self,message): - r=[] - l=self.get_level() + def _make_message(self, level, message): + r = [] for m in message: - if type(m).__name__=='str': + if type(m).__name__ == 'str': r.append(m) else: - r.append(json.dumps(m,sort_keys=True,indent=2)) - return '[' + str(time()) + '] ' + msg_level[l] + ': ' + ''.join(r) + "\n" + r.append(json.dumps(m, sort_keys=True, indent=2)) + return '[' + str(time.time()) + '] ' + self._msg_level[level] + ': ' + ''.join(r) + "\n" - def debug(self,*message): + def debug(self, *message): if self.get_level() in ['debug']: - sys.stderr.write(self.__make_message(message)) + self._log2err(self._make_message('debug',message)) - def info(self,*message): - if self.get_level() in ['debug','info']: - sys.stdout.write(self.__make_message(message)) + def info(self, *message): + if self.get_level() in ['debug', 'info']: + self._log2out(self._make_message('info',message)) - def warning(self,*message): - if self.get_level() in ['debug','info','warning']: - sys.stderr.write(self.__make_message(message)) + def warning(self, *message): + if self.get_level() in ['debug', 'info', 'warning']: + self._log2out(self._make_message('warning',message)) - def error(self,*message): - sys.stderr.write(self.__make_message(message)) + def error(self, *message): + self._log2err(self._make_message('error',message)) - def testing(self,level,message): - self.set_level(level) - return self.__make_message(message) + def _log2err(self,out): + sys.stderr.write(out) + def _log2out(self,out): + sys.stdout.write(out) diff --git a/surok/system.py b/surok/system.py index 89b1466..81717b5 100644 --- a/surok/system.py +++ b/surok/system.py @@ -3,6 +3,7 @@ import sys import requests from .discovery import Discovery from .logger import Logger + logger=Logger() # Get old configuration @@ -91,7 +92,8 @@ def discovery_memcached(conf): # !!! NEED REFACTORING !!! def reload_conf(service_conf, app_conf, conf, app_hosts): # Check marathon enabled in configuration - if conf['marathon'].get('restart',False): + if (conf.get('version','0.7')=='0.8' and conf['marathon'].get('restart',False)) or ( + conf.get('version','0.7')=='0.7' and conf['marathon'].get('enable',False)): if get_old(app_conf['conf_name'], service_conf) != 1: restart_self_in_marathon(conf['marathon']) diff --git a/surok_fedora.spec b/surok_fedora.spec index 57f839f..ee5373f 100644 --- a/surok_fedora.spec +++ b/surok_fedora.spec @@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates} install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf -install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d +install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates @@ -43,7 +45,9 @@ rm -rf $RPM_BUILD_ROOT /opt/surok/surok/discovery.py /opt/surok/surok/system.py /opt/surok/surok/templates.py -/etc/surok/conf.d/selfcheck.json +/opt/surok/surok/config.py +/etc/surok/conf.d/self_check.json +/etc/surok/conf.d/marathon_check.json /etc/surok/templates/selfcheck.jj2 %defattr(-,root,root,-) %doc From 62cb5c24ba13344588517b4c381e600c7867e506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D1=8C=D0=B5=D0=B2=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=92=D0=BB=D0=B0=D0=B4?= =?UTF-8?q?=D0=B8=D0=BC=D0=B8=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 7 Feb 2017 01:58:13 +0300 Subject: [PATCH 05/10] PEP8 --- conf.d/self_check.json | 19 +++-- conf/surok.json | 4 +- surok.py | 10 +-- surok/config.py | 100 +++++++++++++----------- surok/discovery.py | 171 ++++++++++++++++++++++------------------- surok/logger.py | 14 ++-- 6 files changed, 174 insertions(+), 144 deletions(-) diff --git a/conf.d/self_check.json b/conf.d/self_check.json index 4ce08e7..4cf0f65 100644 --- a/conf.d/self_check.json +++ b/conf.d/self_check.json @@ -1,15 +1,20 @@ { "services": [ - {"name":"zzy0", - "ports":["tname_aa","tname_ab","tname_ba","tname_bb","tname_d"] + { + "name": "zzy0", + "ports": ["tname_aa", "tname_ab", "tname_ba", "tname_bb", "tname_d"] }, - {"name":"zzy1", - "ports":["tname_aa","tname_ab","tname_ba","tname_bb"] + { + "name": "zzy1", + "ports": ["tname_aa", "tname_ab", "tname_ba", "tname_bb"] }, - {"name":"zzz0", - "ports":["tname_aa","tname_bb"] + { + "name": "zzz0", + "ports": ["tname_aa", "tname_bb"] }, - {"name":"zzz1"} + { + "name": "zzz1" + } ], "template": "templates/selfcheck.jj2", "dest": "selfcheck", diff --git a/conf/surok.json b/conf/surok.json index 20bd0fd..f93bdb4 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,12 +1,12 @@ { - "version":"0.8", + "version": "0.8", "marathon": { "enabled": false, "restart": false, "force": true, "host": "http://marathon.mesos:8080" }, - "mesos":{ + "mesos": { "enabled": false, "domain": "marathon.mesos" }, diff --git a/surok.py b/surok.py index e9835f2..8ea7fb0 100755 --- a/surok.py +++ b/surok.py @@ -12,7 +12,7 @@ from surok.system import reload_conf from surok.logger import Logger from surok.config import Config -logger=Logger() +logger = Logger() # Command line arguments parser = argparse.ArgumentParser() @@ -20,12 +20,12 @@ parser.add_argument('-c', '--config', help='surok.json path') args = parser.parse_args() # Load base configurations -config=Config(args.config if args.config else '/etc/surok/conf/surok.json') +config = Config(args.config if args.config else '/etc/surok/conf/surok.json') # Main loop -########### +# -discovery=Discovery() +discovery = Discovery() while 1: # Update config from discovery object @@ -38,7 +38,7 @@ while 1: my = {"services": app_hosts, "conf_name": app['conf_name']} - logger.debug('my=',my) + logger.debug('my=', my) # Generate config from template service_conf = gen(my, app['template']) diff --git a/surok/config.py b/surok/config.py index 4d58742..9e73ef4 100644 --- a/surok/config.py +++ b/surok/config.py @@ -23,7 +23,6 @@ type_par - additional parameters for test ''' - ''' Public Config object ================================================== @@ -36,16 +35,18 @@ Public Config object .apps - Apps object. List of AppConfig oblects ''' + class _ConfigTemplate(dict): - _conf={} - def _init_conf(self,params): - conf={} + _conf = {} + + def _init_conf(self, params): + conf = {} for k in params.keys(): if params[k].get('params'): - conf[k]=self._init_conf(params[k].get('params')) + conf[k] = self._init_conf(params[k].get('params')) else: if params[k].get('value') is not None: - conf[k]=params[k].get('value') + conf[k] = params[k].get('value') return conf def __init__(self, *conf_data): @@ -55,49 +56,56 @@ class _ConfigTemplate(dict): for c in conf_data: self.set_config(c) - def _set_conf_params(self,oldconf,testconf,params): + def _set_conf_params(self, oldconf, testconf, params): conf = oldconf if oldconf else {} for key in testconf.keys(): - resvalue=None - param=params.get(key) - oldvalue=conf.get(key) - testvalue=testconf.get(key) + resvalue = None + param = params.get(key) + oldvalue = conf.get(key) + testvalue = testconf.get(key) if param is None: - self._logger.error('Parameter "', key, '" value "', testvalue, '" type is "', type(testvalue).__name__, '" not found') + self._logger.error('Parameter "', key, '" value "', testvalue, + '" type is "', type(testvalue).__name__, '" not found') else: - type_param=param.get('type') - resvalue=[] + type_param = param.get('type') + resvalue = [] if type(testvalue).__name__ != 'list': - testvalue=[testvalue] + testvalue = [testvalue] for testitem in testvalue: if self._test_value(key, testitem, param): if 'dict' in type_param: if param.get('params'): - res=self._set_conf_params(oldvalue,testitem,param.get('params')) + res = self._set_conf_params( + oldvalue, testitem, param.get('params')) if res is not None: resvalue.append(res) else: resvalue.append(testitem) if 'list' not in type_param: - resvalue=list([None]+resvalue).pop() + resvalue = list([None] + resvalue).pop() if resvalue is not None and 'do' in type_param: if not self._do_type_set(key, resvalue, param): - self._logger.warning('Parameter "', key, '" current "', resvalue, '" type is "', type(resvalue).__name__, '" testing failed') + self._logger.warning( + 'Parameter "', key, '" current "', resvalue, '" type is "', type(resvalue).__name__, '" testing failed') resvalue = None if resvalue is not None: - conf[key]=resvalue + conf[key] = resvalue return conf def _test_value(self, key, value, param): - type_param=param.get('type') - type_value=[x for x in type_param if x in ['str', 'int', 'bool', 'dict']] + type_param = param.get('type') + type_value = [ + x for x in type_param if x in ['str', 'int', 'bool', 'dict']] if type_value: if type(value).__name__ not in type_value: - self._logger.error('Parameter "', key, '" must be ', type_value,' types, current "', value, '" (',type(value).__name__,')') + self._logger.error( + 'Parameter "', key, '" must be ', type_value, + ' types, current "', value, '" (', type(value).__name__, ')') return False if 'value' in type_param: - if value not in param.get('values',[]): - self._logger.error('Value "', value, '" of key "', key, '" unknown') + if value not in param.get('values', []): + self._logger.error( + 'Value "', value, '" of key "', key, '" unknown') return False if 'dir' in type_param: if not os.path.isdir(value): @@ -109,7 +117,8 @@ class _ConfigTemplate(dict): return False return True else: - self._logger.error('Type for testing "{}" unknown'.format(type_value)) + self._logger.error( + 'Type for testing "{}" unknown'.format(type_value)) return False def set_config(self, conf_data): @@ -118,7 +127,7 @@ class _ConfigTemplate(dict): try: self._logger.debug('Open file ', conf_data) f = open(conf_data, 'r') - json_data=f.read() + json_data = f.read() f.close() conf = json.loads(json_data) except OSError as err: @@ -134,7 +143,7 @@ class _ConfigTemplate(dict): conf = conf_data else: return False - self._conf=self._set_conf_params(self._conf,conf,self._params) + self._conf = self._set_conf_params(self._conf, conf, self._params) self._logger.debug('Conf=', self._conf) def keys(self): @@ -151,7 +160,7 @@ class _ConfigTemplate(dict): return hashlib.sha1(json.dumps(self._conf, sort_keys=True).encode()).hexdigest() def set(self, key, value): - self._conf[key]=value + self._conf[key] = value def __setitem__(self, key, value): self.set(key, value) @@ -196,7 +205,7 @@ class Config(_ConfigTemplate): 'type': ['bool'] } }, - 'type':['dict'] + 'type': ['dict'] }, 'mesos': { 'params': { @@ -209,7 +218,7 @@ class Config(_ConfigTemplate): 'type': ['bool'] } }, - 'type':['dict'] + 'type': ['dict'] }, 'memcached': { 'params': { @@ -232,23 +241,23 @@ class Config(_ConfigTemplate): 'type': ['str'] } }, - 'type':['dict'] + 'type': ['dict'] }, 'hosts': { 'value': ['localhost:11211'], - 'type': ['list','str'] + 'type': ['list', 'str'] } }, - 'type':['dict'] + 'type': ['dict'] }, 'version': { 'value': '0.7', - 'type': ['str','value'], + 'type': ['str', 'value'], 'values': ['0.7', '0.8'] }, 'confd': { 'value': '/etc/surok/conf.d', - 'type': ['str','dir'] + 'type': ['str', 'dir'] }, 'wait_time': { 'value': 20, @@ -256,7 +265,7 @@ class Config(_ConfigTemplate): }, 'lock_dir': { 'value': '/var/tmp', - 'type': ['str','dir'] + 'type': ['str', 'dir'] }, 'default_discovery': { 'value': 'mesos_dns', @@ -265,8 +274,8 @@ class Config(_ConfigTemplate): }, 'loglevel': { 'value': 'info', - 'type': ['str','do'], - 'do':'set_loglevel' + 'type': ['str', 'do'], + 'do': 'set_loglevel' } } @@ -277,7 +286,7 @@ class Config(_ConfigTemplate): return _config_singleton def __init__(self, *conf_data): - super().__init__( *conf_data) + super().__init__(*conf_data) self.apps = _Apps() def set_config(self, conf_data): @@ -317,7 +326,7 @@ class _Apps: self._apps[app.get('conf_name')] = app def reset(self): - keys=[]+list(self.keys()) + keys = [] + list(self.keys()) for k in keys: del self._apps[k] @@ -361,7 +370,7 @@ class AppConfig(_ConfigTemplate): 'type': ['str'] }, 'ports': { - 'type': ['list','str'] + 'type': ['list', 'str'] }, 'discovery': { 'type': ['str'] @@ -370,10 +379,10 @@ class AppConfig(_ConfigTemplate): 'type': ['str'] } }, - 'type':['list','dict'] + 'type': ['list', 'dict'] }, 'template': { - 'type': ['str','file'] + 'type': ['str', 'file'] }, 'dest': { 'type': ['str'] @@ -395,13 +404,14 @@ class AppConfig(_ConfigTemplate): def set_config(self, conf_data): super().set_config(conf_data) - self._conf.setdefault('discovery', self._config.get('default_discovery')) + self._conf.setdefault( + 'discovery', self._config.get('default_discovery')) self._conf.setdefault('group', self._get_default_group()) if type(conf_data).__name__ == 'str': self._conf.setdefault('conf_name', os.path.basename(conf_data)) def _get_default_group(self): - env=self._config.get('env',dict(os.environ)) + env = self._config.get('env', dict(os.environ)) # Check environment variable if env.get('SUROK_DISCOVERY_GROUP'): return env['SUROK_DISCOVERY_GROUP'] diff --git a/surok/discovery.py b/surok/discovery.py index 8018b7f..25bad3c 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -1,5 +1,5 @@ -#Public names -__all__=['Discovery','DiscoveryMesos','DiscoveryMarathon'] +# Public names +__all__ = ['Discovery', 'DiscoveryMesos', 'DiscoveryMarathon'] import dns.resolver import dns.query import os @@ -10,15 +10,17 @@ from .config import * from .logger import * # Discovery object -_discovery_singleton=None +_discovery_singleton = None + class DiscoveryTemplate: + def __init__(self): - self._config=Config() - self._logger=Logger() + self._config = Config() + self._logger = Logger() def enabled(self): - return self._config[self._config_section].get('enabled',False) + return self._config[self._config_section].get('enabled', False) def update_data(self): pass @@ -26,20 +28,20 @@ class DiscoveryTemplate: # Do DNS queries # Return array: # ["10.10.10.1", "10.10.10.2"] - def do_query_a(self,fqdn): + def do_query_a(self, fqdn): servers = [] try: resolver = dns.resolver.Resolver() for a_rdata in resolver.query(fqdn, 'A'): servers.append(a_rdata.address) except DNSException as e: - self._logger.error("Could not resolve ",fqdn) + self._logger.error("Could not resolve ", fqdn) return servers # Do DNS queries # Return array: # [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}] - def do_query_srv(self,fqdn): + def do_query_srv(self, fqdn): servers = [] try: resolver = dns.resolver.Resolver() @@ -50,38 +52,39 @@ class DiscoveryTemplate: info = str(rdata).split() servers.append({'name': info[3][:-1], 'port': info[2]}) except DNSException as e: - self._logger.warning("Could not resolve ",fqdn) + self._logger.warning("Could not resolve ", fqdn) return servers class Discovery: - _discoveries={} + _discoveries = {} + def __new__(cls): global _discovery_singleton if _discovery_singleton is None: - _discovery_singleton=super(Discovery, cls).__new__(cls) + _discovery_singleton = super(Discovery, cls).__new__(cls) return _discovery_singleton def __init__(self): - self._config=Config() - self._logger=Logger() + self._config = Config() + self._logger = Logger() if not self._discoveries.get('mesos_dns'): - self._discoveries['mesos_dns']=DiscoveryMesos() + self._discoveries['mesos_dns'] = DiscoveryMesos() if not self._discoveries.get('marathon_api'): - self._discoveries['marathon_api']=DiscoveryMarathon() + self._discoveries['marathon_api'] = DiscoveryMarathon() def keys(self): return self._discoveries.keys() - def resolve(self,app): - discovery=app.get('discovery',self._config.get('default_discovery')) + def resolve(self, app): + discovery = app.get('discovery', self._config.get('default_discovery')) if discovery not in self.keys(): - self._logger.warning('Discovery "',discovery,'" is not present') + self._logger.warning('Discovery "', discovery, '" is not present') return {} if self._discoveries[discovery].enabled(): return self.compatible(self._discoveries[discovery].resolve(app)) else: - self._logger.error('Discovery "',discovery,'" is disabled') + self._logger.error('Discovery "', discovery, '" is disabled') return {} def update_data(self): @@ -90,128 +93,140 @@ class Discovery: if self._discoveries[d].enabled(): self._discoveries[d].update_data() - def compatible(self,hosts): - compatible_hosts={} + def compatible(self, hosts): + compatible_hosts = {} if self._config.get('version') == '0.7': for service in hosts.keys(): for host in hosts[service]: - ports=host.get('tcp',[]) + ports = host.get('tcp', []) if type(ports).__name__ == 'list': - compatible_hosts[service]=[] + compatible_hosts[service] = [] for port in ports: - compatible_hosts[service].append({'name':host['name'], - 'ip':host['ip'], - 'port':str(port)}) + compatible_hosts[service].append( + {'name': host['name'], + 'ip': host['ip'], + 'port': str(port)}) else: - compatible_hosts[service]={} + compatible_hosts[service] = {} for port in ports.keys(): - compatible_host=compatible_hosts[service].setdefault(port,[]) - compatible_host.append({'name':host['name'], - 'ip':host['ip'], - 'port':ports[port]}) + compatible_host = compatible_hosts[ + service].setdefault(port, []) + compatible_host.append({'name': host['name'], + 'ip': host['ip'], + 'port': ports[port]}) return compatible_hosts return hosts class DiscoveryMesos(DiscoveryTemplate): - _config_section='mesos' - def resolve(self,app): + _config_section = 'mesos' + + def resolve(self, app): hosts = {} services = app.get('services') domain = self._config[self._config_section].get('domain') for service in services: - group = service.get('group',app.get('group')) + group = service.get('group', app.get('group')) if group is None: - self._logger.error('Group for service "{}" of config "{}" not found'.format(service['name'],app.get('conf_name'))) + self._logger.error('Group for service "{}" of config "{}" not found'.format( + service['name'], app.get('conf_name'))) continue ports = service.get('ports') name = service['name'] hosts[name] = {} serv = hosts[name] - self._logger.debug('group=',group,' ports=',ports,' name=',name,' serv=',serv) - for prot in ['tcp','udp']: + self._logger.debug( + 'group=', group, ' ports=', ports, ' name=', name, ' serv=', serv) + for prot in ['tcp', 'udp']: if ports is not None: for port_name in ports: - for d in self.do_query_srv('_'+port_name+'._'+name+'.'+group+'._'+prot+'.'+domain): - hostname=d['name'] - serv.setdefault(hostname,{'name':hostname, - 'ip':self.do_query_a(hostname)}) - serv[hostname].setdefault(prot,{}) - serv[hostname][prot][port_name]=d['port'] + for d in self.do_query_srv('_' + port_name + '._' + name + '.' + group + '._' + prot + '.' + domain): + hostname = d['name'] + serv.setdefault(hostname, {'name': hostname, + 'ip': self.do_query_a(hostname)}) + serv[hostname].setdefault(prot, {}) + serv[hostname][prot][port_name] = d['port'] else: - for d in self.do_query_srv('_'+name+'.'+group+'._'+prot+'.'+domain): - hostname=d['name'] + for d in self.do_query_srv('_' + name + '.' + group + '._' + prot + '.' + domain): + hostname = d['name'] if serv.get(hostname) is None: - serv[hostname]={'name':hostname, - 'ip':self.do_query_a(hostname)} + serv[hostname] = {'name': hostname, + 'ip': self.do_query_a(hostname)} if serv[hostname].get(prot) is None: - serv[hostname][prot]=[] + serv[hostname][prot] = [] serv[hostname][prot].extend([d['port']]) - hosts[name]=list(serv.values()) + hosts[name] = list(serv.values()) return hosts class DiscoveryMarathon(DiscoveryTemplate): - _config_section='marathon' + _config_section = 'marathon' _tasks = [] _ports = {} def update_data(self): - hostname=self._config[self._config_section].get('host') + hostname = self._config[self._config_section].get('host') try: ports = {} - for app in requests.get(hostname+'/v2/apps').json()['apps']: + for app in requests.get(hostname + '/v2/apps').json()['apps']: ports[app['id']] = {} if app.get('container') is not None and app['container'].get('type') == 'DOCKER': - ports[app['id']] = app['container']['docker'].get('portMappings',[]) - self._ports=ports + ports[app['id']] = app['container'][ + 'docker'].get('portMappings', []) + self._ports = ports except: - self._logger.warning('Apps (',hostname,'/v2/apps) request from Marathon API is failed') + self._logger.warning( + 'Apps (', hostname, '/v2/apps) request from Marathon API is failed') pass try: self._tasks = requests.get(hostname + '/v2/tasks').json()['tasks'] except: - self._logger.warning('Tasks (',hostname,'/v2/tasks) request from Marathon API is failed') + self._logger.warning( + 'Tasks (', hostname, '/v2/tasks) request from Marathon API is failed') pass def _test_mask(self, mask, value): return (mask.endswith('*') and value.startswith(mask[:-1])) or mask == value def resolve(self, app): - hosts={} + hosts = {} services = app.get('services') if not services: - services = [{'name':'*','ports':['*']}] + services = [{'name': '*', 'ports': ['*']}] for service in services: # Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format - group=service.get('group',app.get('group')) + group = service.get('group', app.get('group')) if group is None: - self._logger.error('Group for service "{}" of config "{}" not found'.format(service['name'],app.get('conf_name'))) + self._logger.error('Group for service "{}" of config "{}" not found'.format( + service['name'], app.get('conf_name'))) continue - group = '/'+'/'.join(group.split('.')[::-1])+'/' + group = '/' + '/'.join(group.split('.')[::-1]) + '/' service_mask = group + service['name'] for task in self._tasks: - if self._test_mask(service_mask,task['appId']): - name='.'.join(task['appId'][len(group):].split('/')[::-1]) - hosts[name]={} + if self._test_mask(service_mask, task['appId']): + name = '.'.join( + task['appId'][len(group):].split('/')[::-1]) + hosts[name] = {} serv = hosts[name] - hostname=task['host'] + hostname = task['host'] for task_port in self._ports[task['appId']]: - prot=task_port['protocol'] - port_name=task_port['name'] - port=task['ports'][task['servicePorts'].index(task_port['servicePort'])] + prot = task_port['protocol'] + port_name = task_port['name'] + port = task['ports'][ + task['servicePorts'].index(task_port['servicePort'])] if 'ports' in service: for port_mask in service['ports']: - if self._test_mask(port_mask,port_name): - serv.setdefault(hostname,{'name':hostname, - 'ip':self.do_query_a(hostname)}) - serv[hostname].setdefault(prot,{}) - serv[hostname][prot][port_name]=port + if self._test_mask(port_mask, port_name): + serv.setdefault( + hostname, {'name': hostname, + 'ip': self.do_query_a(hostname)}) + serv[hostname].setdefault(prot, {}) + serv[hostname][prot][port_name] = port else: - serv.setdefault(hostname,{'name':hostname, - 'ip':self.do_query_a(hostname)}) - serv[hostname].setdefault(prot,[]) + serv.setdefault(hostname, {'name': hostname, + 'ip': self.do_query_a(hostname)}) + serv[hostname].setdefault(prot, []) serv[hostname][prot].extend([port]) - hosts[name]=list(serv.values()) + hosts[name] = list(serv.values()) return hosts diff --git a/surok/logger.py b/surok/logger.py index eec8efb..1bf2fa6 100644 --- a/surok/logger.py +++ b/surok/logger.py @@ -28,7 +28,7 @@ Public Logger oblect class Logger: _loglevel = 'info' _msg_level = { - 'debug': 'DEBUG', + 'debug': 'DEBUG', 'info': 'INFO', 'warning': 'WARNING', 'error': 'ERROR' @@ -66,21 +66,21 @@ class Logger: def debug(self, *message): if self.get_level() in ['debug']: - self._log2err(self._make_message('debug',message)) + self._log2err(self._make_message('debug', message)) def info(self, *message): if self.get_level() in ['debug', 'info']: - self._log2out(self._make_message('info',message)) + self._log2out(self._make_message('info', message)) def warning(self, *message): if self.get_level() in ['debug', 'info', 'warning']: - self._log2out(self._make_message('warning',message)) + self._log2out(self._make_message('warning', message)) def error(self, *message): - self._log2err(self._make_message('error',message)) + self._log2err(self._make_message('error', message)) - def _log2err(self,out): + def _log2err(self, out): sys.stderr.write(out) - def _log2out(self,out): + def _log2out(self, out): sys.stdout.write(out) From 53f446e219fc5e24c7497fcfacc529426859a49b Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Tue, 7 Feb 2017 12:41:43 +0300 Subject: [PATCH 06/10] Docs update --- docs/App-config-file.md | 19 ++--- docs/Main-config-file.md | 67 ++++++++++++--- docs/Templates.md | 179 +++++++++++++++++++++++++++++++-------- 3 files changed, 208 insertions(+), 57 deletions(-) diff --git a/docs/App-config-file.md b/docs/App-config-file.md index 5156e5a..34963d6 100644 --- a/docs/App-config-file.md +++ b/docs/App-config-file.md @@ -5,19 +5,16 @@ conf.d/myapp.json { "services": [ { - "name": "myapp", - "group": "backend.production", - "ports": ["proxy", "api"] - }, - { - "name": "nginx", - "group": "frontend.production" + "name": "kioskservice", + "group": "production.romania", + "ports": ["web", "socket"] } ], - "conf_name": "myapp_backend_production", - "template": "/etc/surok/templates/myapp.jj2", - "dest": "/etc/myapp/myapp.cfg", - "reload_cmd": "killall -9 myapp; /usr/local/bin/myapp -config /etc/myapp/myapp.cfg" + "conf_name": "kiosk", + "template": "/etc/surok/templates/kiosk.jj2", + "dest": "/etc/nginx/sites-available/kioskservice.conf", + "reload_cmd": "/sbin/nginx -t && /bin/systemctl reload nginx", + "run_cmd": ["/usr/bin/node", "-c", "config.json"] } ``` * **services**. List of hashes with required services for app. diff --git a/docs/Main-config-file.md b/docs/Main-config-file.md index db2efd6..60f33ab 100644 --- a/docs/Main-config-file.md +++ b/docs/Main-config-file.md @@ -1,10 +1,10 @@ -# Surok main config file +# Surok main config file (0.8.x) -Default location is /etc/surok/conf/surok.json +Default location is **/etc/surok/conf/surok.json** -conf/surok.json ``` { + "version": "0.8" "marathon": { "enabled": false, "restart": false, @@ -24,7 +24,6 @@ conf/surok.json "wait_time": 20, "lock_dir": "/var/tmp", "loglevel": "info", - "container": false, "memcached": { "enabled": false, "discovery": { @@ -36,10 +35,56 @@ conf/surok.json } } ``` -* **marathon section**. Restarting app over marathon api if config changed. Disabled by default. -* **confd**. Directory where located configs apps. -* **domain**. Domain served by mesos-dns. -* **lock_dir**. Directory where surok writes temporary configs after resolving. -* **wait_time**. Sleep time in main loop. -* **container**. Not implemented. -* **memcached section**. Memcached support. Disabled by default. + +## Config file options +* **version** - *string. Optional. "0.7" by default.* +Config files and templates version. Accept "0.7" or "0.8". + * "0.7" - config files <= 0.7.х version + * "0.8" - >= 0.8.x config files version + +##### 0.8 version +* **marathon**, **mesos**, **consul**, **memcached** - *dict/hash. Optional. '{"enable":false}'. by default* +Surok working with folowing systems. If system is disabled parameters will be ignored. + * **enable** - *boolean. Optional. false by default* + Enable/disable system for usage. + + Specific variables: + * For Marathon API "marathon" + * **force** - *boolean. Optional. true by default* + Force restart container over API. + * **restart** - *boolean. Optional. false by default* + Enable/disable restarting container + * **host** - *string. Optional. "http://marathon.mesos:8080" by default* + Marathon address. + * For Consul "consul" + * **domain** - *string. Required.* + Consul private domain + * For mesos-dns "mesos" + * **domain** - *string. Optional. "marathon.mesos" by default* + mesos-dns private domain + * For Memcached "memcached" + * **hosts** - memcached hosts + * **discovery** + * **enabled** - boolean. Enable/disable disovery memcached service + * **service** - string. memcached app name + * **group** - string. memcached app group +* **default_discovery** - *string. Optional. "mesos_dns" by default* + Accept values: + * "mesos_dns" - mesos-dns + * "marathon_api"- Marathon API + * "consul_dns" - Consul +* **confd** - *strig. Required.* + Path to directory with app config files. +* **wait_time** - *int. Required.* + Time in seconds how much Surok waits before starting to re-do the requests for service discovery +* **lock_dir** - *string. Required.* + Path to directory where Surok write lock-files. +* **loglevel** - *string. Optional. "info" by default* + Logleve. Accept values: "debug", "info", "warning", "error" + +##### < 0.8 versions + +* **marathon** + * **enabled** - boolean. Enable/disable container restart. Renamed to "restart" in 0.8 version. +* **domain** - string. mesos-dns private domain. Moved to "mesos" hashtable in 0.8 version. + Discovery over mesos-dns enabled all times. diff --git a/docs/Templates.md b/docs/Templates.md index 50ea7d5..90d2431 100644 --- a/docs/Templates.md +++ b/docs/Templates.md @@ -5,8 +5,11 @@ - [Templates](#templates) - [my dictionary in templates](#my-dictionary-in-templates) - - [Real template example](#real-template-example) - - [Checks in template](#checks-in-template) + - [0.8 version](#08-version) + - [Real template example](#real-template-example) + - [Checks in template](#checks-in-template) + - [0.7 version](#07-version) + - [Real template example](#real-template-example) @@ -15,55 +18,74 @@ Surok using Jinja2 for templates. [Jinja2 documentation](http://jinja.pocoo.org/ ## my dictionary in templates +### 0.8 version + ``` { "services": { - "nginx": [ + "asterisk": [ { - "name": "nginx.testing-kl92-s0.marathon.mesos.", - "port": "31200", - "ip": ["10.10.10.1"] + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1", + "11.0.0.1" + ], + "tcp": { + "rpc":31200, + "web":31201, + "sip":32000 + }, + "udp": { + "sip":31201 + } }, { - "name": "nginx.testing-kl123-s1.marathon.mesos.", - "port": "32230", - "ip": ["10.10.10.2"] + "name": "nginx.testing-kl123-s1.marathon.mesos.", + "ip": [ + "10.0.0.2", + "11.0.0.2" + ], + "tcp": { + "rpc":31210, + "web":31211, + "sip":32010 + }, + "udp": { + "sip":31211 + } } ], - "emailsender": [ + "email": [ { - "name": "emailsender.testing-kl92-s0.marathon.mesos.", - "port": "31201", - "ip": ["10.10.10.1"] - }, - { - "name": "emailsender.testing-kl123-s1.marathon.mesos.", - "port": "32232", - "ip": ["10.10.10.1"] + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1" + ], + "tcp": { + "smtp":31200, + "pop":31201 + } } ], - "service-with-defined-ports": { - "web": [ - { - "name": "f.q.d.n", - "port": 12341 - } - ], - "rpc": [ - { - "name": "f.q.d.n", - "port": 12342 - } - ] - } - }, + "anyport": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "ip": [ + "10.0.0.1" + ], + "tcp": [ + 31200, + 31201 + ] + } + ] "env": { "HOME": "/var/lib/nginx" } } ``` -## Real template example +#### Real template example nginx config ``` @@ -104,7 +126,7 @@ server { } ``` -## Checks in template +#### Checks in template _my['env']_ is a python os.environ class. Look bellow: ``` @@ -114,3 +136,90 @@ host = '{{my['env']['DB_HOST']}}' host = 'localhost' {% endif %} ``` + +### 0.7 version + +my dictionary in template +``` +{ + "services": { + "nginx": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "port": "31200" + }, + { + "name": "nginx.testing-kl123-s1.marathon.mesos.", + "port": "32230" + } + ], + "emailsender": [ + { + "name": "emailsender.testing-kl92-s0.marathon.mesos.", + "port": "31201" + }, + { + "name": "emailsender.testing-kl123-s1.marathon.mesos.", + "port": "32232" + } + ], + "service-with-defined-ports": { + "name-of-port0": [ + { + "name": "f.q.d.n", + "port": 12341 + } + ], + "name-of-port2": [ + { + "name": "f.q.d.n", + "port": 12342 + } + ] + } + }, + "env": { + "HOME": "/var/lib/nginx" + } +} +``` + +#### Real template example + +``` +upstream matrix-http { + hash $remote_addr; +{% for server in my['services']['matrix']['http'] %} + server {{server['name']}}:{{server['port']}} max_fails=3; +{% endfor %} +} + +upstream riot-http { + hash $remote_addr; +{% for server in my['services']['riot'] %} + server {{server['name']}}:{{server['port']}} max_fails=3; +{% endfor %} +} + +server { + listen 10.15.56.157:80; + server_name matrix.example.com; + + client_max_body_size 10m; + + location / { + proxy_pass http://riot-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /_matrix/ { + proxy_pass http://matrix-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + +} +``` From d7c0ddf299544f6f57d068fada7345f9a49297cc Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Tue, 7 Feb 2017 12:47:12 +0300 Subject: [PATCH 07/10] Version bump. Release commit --- debian/changelog | 11 ++++++++++- surok.spec | 6 +++++- surok_fedora.spec | 6 +++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7d14ccc..c0c0e08 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +surok (0.8) testing; urgency=medium + + * Version bump + * new config structure + * Basic consul support + * Discovery over Marathon API + + -- Denis Zheleztsov Tue, 07 Feb 2017 12:41:52 +0300 + surok (0.7.4.3) testing; urgency=medium * Fixed #9 @@ -11,7 +20,7 @@ surok (0.7.4.3) testing; urgency=medium surok (0.7.4.1) testing; urgency=medium * closes #3 - * Fixed bug with very long dns query lifetime + * Fixed bug with very long dns query lifetime -- Denis Zheleztsov Fri, 11 Nov 2016 21:15:02 +0300 diff --git a/surok.spec b/surok.spec index 33c1a53..c7fa158 100644 --- a/surok.spec +++ b/surok.spec @@ -1,6 +1,6 @@ Summary: Simple service discovery for Apache Mesos clusters Name: surok -Version: 0.7.4.3 +Version: 0.8 Release: 1 License: BSD Group: admin @@ -73,6 +73,10 @@ cd /opt/surok && pip3 install -r requriments.txt %changelog +* Tue Feb 7 2017 Denis Zheleztsov +- New major release +- Discovery over marathon api +- Basic Consul support * Mon Nov 14 2016 Denis Zheleztsov - - Initial build. diff --git a/surok_fedora.spec b/surok_fedora.spec index ee5373f..f67c8a1 100644 --- a/surok_fedora.spec +++ b/surok_fedora.spec @@ -1,6 +1,6 @@ Summary: Simple service discovery for Apache Mesos clusters Name: surok -Version: 0.7.4.3 +Version: 0.8 Release: 1.fc24 License: BSD Group: admin @@ -54,6 +54,10 @@ rm -rf $RPM_BUILD_ROOT %changelog +* Tue Feb 7 2017 Denis Zheleztsov +- New major release +- Discovery over marathon api +- Basic Consul support * Mon Nov 14 2016 Denis Zheleztsov - - Initial build. From 3d00f4b71384a016d91748f2758c77929452b22e Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Tue, 7 Feb 2017 13:03:46 +0300 Subject: [PATCH 08/10] Fix docs and changelog --- debian/changelog | 2 +- docs/Main-config-file.md | 8 -------- docs/ru/surok.configuration.md | 12 ------------ surok.spec | 1 - surok_fedora.spec | 1 - 5 files changed, 1 insertion(+), 23 deletions(-) diff --git a/debian/changelog b/debian/changelog index c0c0e08..7ce8512 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,7 +2,7 @@ surok (0.8) testing; urgency=medium * Version bump * new config structure - * Basic consul support + * Port type * Discovery over Marathon API -- Denis Zheleztsov Tue, 07 Feb 2017 12:41:52 +0300 diff --git a/docs/Main-config-file.md b/docs/Main-config-file.md index 60f33ab..cd75b99 100644 --- a/docs/Main-config-file.md +++ b/docs/Main-config-file.md @@ -11,10 +11,6 @@ Default location is **/etc/surok/conf/surok.json** "force": true, "host": "http://marathon.mesos:8080" }, - "consul": { - "enabled": false, - "domain": "service.dc1.consul" - }, "mesos":{ "enabled": true, "domain": "marathon.mesos" @@ -56,9 +52,6 @@ Surok working with folowing systems. If system is disabled parameters will be ig Enable/disable restarting container * **host** - *string. Optional. "http://marathon.mesos:8080" by default* Marathon address. - * For Consul "consul" - * **domain** - *string. Required.* - Consul private domain * For mesos-dns "mesos" * **domain** - *string. Optional. "marathon.mesos" by default* mesos-dns private domain @@ -72,7 +65,6 @@ Surok working with folowing systems. If system is disabled parameters will be ig Accept values: * "mesos_dns" - mesos-dns * "marathon_api"- Marathon API - * "consul_dns" - Consul * **confd** - *strig. Required.* Path to directory with app config files. * **wait_time** - *int. Required.* diff --git a/docs/ru/surok.configuration.md b/docs/ru/surok.configuration.md index 754bc5d..973721b 100644 --- a/docs/ru/surok.configuration.md +++ b/docs/ru/surok.configuration.md @@ -9,14 +9,6 @@ "force": true, "host": "http://marathon.mesos:8080" }, - "consul": { - "enabled": false, - "domain": "service.dc1.consul" - }, - "mesos":{ - "enabled": true, - "domain": "marathon.mesos" - }, "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", "wait_time": 20, @@ -53,9 +45,6 @@ Вкл/выкл. рестарта контейнера * **host** - *string. Не обязательный. По умолчанию "http://marathon.mesos:8080".* Адрес Marathon. - * для Consul "consul" - * **domain** - *string. Обязательный.* - Приватный домен Consul * для mesos DNS "mesos" * **domain** - *string. Не обязательный. По умолчанию "marathon.mesos".* Приватный домен Mesos DNS @@ -70,7 +59,6 @@ Может принимать значения: * "mesos_dns" - Mesos DNS * "marathon_api"- Marathon API - * "consul_dns" - Consul * **confd** - *strig. Обязательный.* Абсолютный путь до директории с конфигурационными файлами приложений. * **wait_time** - *int. Обязательный.* diff --git a/surok.spec b/surok.spec index c7fa158..763b6a1 100644 --- a/surok.spec +++ b/surok.spec @@ -76,7 +76,6 @@ cd /opt/surok && pip3 install -r requriments.txt * Tue Feb 7 2017 Denis Zheleztsov - New major release - Discovery over marathon api -- Basic Consul support * Mon Nov 14 2016 Denis Zheleztsov - - Initial build. diff --git a/surok_fedora.spec b/surok_fedora.spec index f67c8a1..d9cbad6 100644 --- a/surok_fedora.spec +++ b/surok_fedora.spec @@ -57,7 +57,6 @@ rm -rf $RPM_BUILD_ROOT * Tue Feb 7 2017 Denis Zheleztsov - New major release - Discovery over marathon api -- Basic Consul support * Mon Nov 14 2016 Denis Zheleztsov - - Initial build. From 0d27591b8bf07fc7746247df83e6d615b58ee813 Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Tue, 7 Feb 2017 13:05:41 +0300 Subject: [PATCH 09/10] revert mesos section --- docs/ru/surok.configuration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/ru/surok.configuration.md b/docs/ru/surok.configuration.md index 973721b..a52283b 100644 --- a/docs/ru/surok.configuration.md +++ b/docs/ru/surok.configuration.md @@ -9,6 +9,10 @@ "force": true, "host": "http://marathon.mesos:8080" }, + "mesos":{ + "enabled": true, + "domain": "marathon.mesos" + }, "default_discovery": "mesos_dns", "confd": "/etc/surok/conf.d", "wait_time": 20, From 6053289048ceed5835a015a99eed5871d970fc7c Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Tue, 7 Feb 2017 13:17:20 +0300 Subject: [PATCH 10/10] Another docs update --- docs/App-config-file.md | 4 ++++ docs/ru/app.configuration.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/App-config-file.md b/docs/App-config-file.md index 34963d6..602b059 100644 --- a/docs/App-config-file.md +++ b/docs/App-config-file.md @@ -10,6 +10,8 @@ conf.d/myapp.json "ports": ["web", "socket"] } ], + "discovery": "mesos_dns", + "group": "dev.web", "conf_name": "kiosk", "template": "/etc/surok/templates/kiosk.jj2", "dest": "/etc/nginx/sites-available/kioskservice.conf", @@ -25,3 +27,5 @@ conf.d/myapp.json * **template**. Jinja2 template location. * **dest**. Destination config path. * **reload_cmd**. Command to execute if generated config is changed. +* **discovery**. Use custom discovery for app. +* **group**. Default group for all required services. diff --git a/docs/ru/app.configuration.md b/docs/ru/app.configuration.md index 11c89d6..5dcba65 100644 --- a/docs/ru/app.configuration.md +++ b/docs/ru/app.configuration.md @@ -10,6 +10,8 @@ "ports": ["web", "socket"] } ], + "discovery": "mesos_dns", + "group": "dev.web", "conf_name": "kiosk", "template": "/etc/surok/templates/kiosk.jj2", "dest": "/etc/nginx/sites-available/kioskservice.conf", @@ -31,3 +33,5 @@ В reload_cmd можно использовать переменные окружения: ```"reload_cmd": "/usr/bin/killall -9 calc || true && /usr/local/bin/calc -c /app/calc.conf ${CALC_NUM}"``` * run_cmd(v0.6) - array. Список с командой на выполнение. Используется внутри контейнера вместо reload_cmd. +* discovery - Опциональный. переопределяет метод обнаружения сервисов для приложения +* group - Опциональный. Группа по-умолчанию.