From 4766ad74eee8648d66fd2e3db7a141c7b5cef07d Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 9 Aug 2016 14:13:20 +0300 Subject: [PATCH 01/39] PEP8 --- surok.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/surok.py b/surok.py index cac072e..47eb8e8 100755 --- a/surok.py +++ b/surok.py @@ -2,7 +2,7 @@ from time import sleep from os import listdir -from os.path import isfile, join +from os.path import isfile, join import json from surok.templates import gen from surok.discovery import resolve @@ -30,15 +30,16 @@ f.close() # Get app configurations # Return list of patches to app discovery configuration def get_configs(): - confs = [f for f in listdir(conf['confd']) if isfile( join(conf['confd'], f) )] + confs = [f for f in listdir(conf['confd']) if isfile( + join(conf['confd'], f))] return confs # Get Surok App configuration # Read app conf from file and return dict def load_app_conf(app): - f = open( conf['confd'] + '/' + app ) - c = json.loads( f.read() ) + f = open(conf['confd'] + '/' + app) + c = json.loads(f.read()) f.close() return c @@ -56,20 +57,17 @@ while 1: for app in confs: app_conf = load_app_conf(app) - # Resolve services + # Resolve services app_hosts = resolve(app_conf, conf) # Populate my dictionary - my = { "services": app_hosts, - "conf_name": app_conf['conf_name'] - } - - # Generate config from template + my = {"services": app_hosts, + "conf_name": app_conf['conf_name']} + + # Generate config from template service_conf = gen(my, app_conf['template']) - stdout, first = reload_conf(service_conf, app_conf, first) + stdout, first = reload_conf(service_conf, app_conf, first) print(stdout) - - sleep( conf['wait_time'] ) - + sleep(conf['wait_time']) From ca0df7c33c3841a76344d392c68bce2593b2e989 Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 9 Aug 2016 15:23:32 +0300 Subject: [PATCH 02/39] PEP8 --- surok/discovery.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/surok/discovery.py b/surok/discovery.py index f4a6ff4..4090213 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -1,5 +1,8 @@ import dns.resolver + +# Resolve service from mesos-dns SRV record +# return dict {"servicename": [{"name": "service.f.q.d.n", "port": 9999}]} def resolve(app, conf): hosts = {} services = app['services'] @@ -7,12 +10,17 @@ def resolve(app, conf): for service in services: hosts[service['name']] = [] try: - for rdata in dns.resolver.query('_' + service['name'] + '.' + service['group'] + '._tcp.' + domain, 'SRV'): + for rdata in dns.resolver.query('_' + + service['name'] + '.' + + service['group'] + '._tcp.' + + domain, 'SRV'): info = str(rdata).split() - server = { 'name': info[3], 'port': info[2] } + server = {'name': info[3], 'port': info[2]} - hosts[ service['name'] ].append(server) + hosts[service['name']].append(server) except Exception as e: - print("Could not resolve " + service['name'] + '.' + service['group'] + '._tcp.' + domain) + print(str(e) + ": Could not resolve " + + service['name'] + '.' + + service['group'] + '._tcp.' + domain) return hosts From 20155298ed450b7ae7b6685e099cf09f96fd1f14 Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 9 Aug 2016 15:23:57 +0300 Subject: [PATCH 03/39] gitignore update --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6e332f6..8b497a9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ surok/__pycache__ __pycache__ *.pyc *.swp +selfcheck From c173a68f5e2ebc4037ea0abddfe0ef9c8fead325 Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 9 Aug 2016 15:40:07 +0300 Subject: [PATCH 04/39] OS environment for app conf --- surok.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/surok.py b/surok.py index 47eb8e8..f7657b9 100755 --- a/surok.py +++ b/surok.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 from time import sleep +import os from os import listdir from os.path import isfile, join import json @@ -38,10 +39,13 @@ def get_configs(): # Get Surok App configuration # Read app conf from file and return dict def load_app_conf(app): + # Load OS environment to app_conf f = open(conf['confd'] + '/' + app) c = json.loads(f.read()) f.close() + c['env'] = os.environ + return c From 9120d723a25b86b02080962fcffb4ed1f59891a2 Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 9 Aug 2016 15:44:23 +0300 Subject: [PATCH 05/39] SUROK_GROUP env variable --- surok/discovery.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/surok/discovery.py b/surok/discovery.py index 4090213..fc28ea1 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -7,12 +7,17 @@ def resolve(app, conf): hosts = {} services = app['services'] domain = conf['domain'] + group = None for service in services: hosts[service['name']] = [] + if app['env'].get('SUROK_GROUP') is not None: + group = app['env']['SUROK_GROUP'] + else: + group = service['group'] try: for rdata in dns.resolver.query('_' + service['name'] + '.' + - service['group'] + '._tcp.' + + group + '._tcp.' + domain, 'SRV'): info = str(rdata).split() server = {'name': info[3], 'port': info[2]} @@ -21,6 +26,6 @@ def resolve(app, conf): except Exception as e: print(str(e) + ": Could not resolve " + service['name'] + '.' + - service['group'] + '._tcp.' + domain) + group + '._tcp.' + domain) return hosts From 2e4758a3e877cd8de278da09cd3e3e1639f1ef7d Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 9 Aug 2016 16:15:45 +0300 Subject: [PATCH 06/39] get discovery group from OS environment --- surok/discovery.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/surok/discovery.py b/surok/discovery.py index fc28ea1..5f93c9b 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -8,12 +8,18 @@ def resolve(app, conf): services = app['services'] domain = conf['domain'] group = None + + if app['env'].get('SUROK_DISCOVERY_GROUP') is not None: + group = app['env']['SUROK_DISCOVERY_GROUP'] + for service in services: hosts[service['name']] = [] - if app['env'].get('SUROK_GROUP') is not None: - group = app['env']['SUROK_GROUP'] + + if group is not None: + pass else: group = service['group'] + try: for rdata in dns.resolver.query('_' + service['name'] + '.' + From 60b1a798bfa59b2914c35665be1a803029fc377e Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 9 Aug 2016 16:34:52 +0300 Subject: [PATCH 07/39] aditional comments --- surok/discovery.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surok/discovery.py b/surok/discovery.py index 5f93c9b..a3d070e 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -15,9 +15,12 @@ def resolve(app, conf): for service in services: hosts[service['name']] = [] + # Check group configuration if group is not None: pass else: + # Load group from service config + # /etc/surok/conf.d/service_conf.json group = service['group'] try: From fe92921cac9e6a7d4c5e0d86024614138aeefedf Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 10 Aug 2016 11:55:18 +0300 Subject: [PATCH 08/39] dns query lifetime --- surok/discovery.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/surok/discovery.py b/surok/discovery.py index a3d070e..cbde725 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -1,8 +1,10 @@ import dns.resolver +import dns.query +from dns.exception import DNSException # Resolve service from mesos-dns SRV record -# return dict {"servicename": [{"name": "service.f.q.d.n", "port": 9999}]} +# return dict {"servicename": [{"name": "service.f.q.d.n.", "port": 9999}]} def resolve(app, conf): hosts = {} services = app['services'] @@ -24,16 +26,16 @@ def resolve(app, conf): group = service['group'] try: - for rdata in dns.resolver.query('_' + - service['name'] + '.' + - group + '._tcp.' + - domain, 'SRV'): + fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain + query = dns.resolver.query(fqdn, 'SRV') + query.lifetime = 1.0 + + for rdata in query: info = str(rdata).split() server = {'name': info[3], 'port': info[2]} - hosts[service['name']].append(server) - except Exception as e: - print(str(e) + ": Could not resolve " + + except DNSException: + print("Could not resolve " + service['name'] + '.' + group + '._tcp.' + domain) From 3fa821e2dc2b0fa1fb6f5a999f318412c2c9dc3a Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 10 Aug 2016 12:07:03 +0300 Subject: [PATCH 09/39] Some refactoring --- surok/discovery.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/surok/discovery.py b/surok/discovery.py index cbde725..689463b 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -25,18 +25,25 @@ def resolve(app, conf): # /etc/surok/conf.d/service_conf.json group = service['group'] - try: - fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain - query = dns.resolver.query(fqdn, 'SRV') - query.lifetime = 1.0 - - for rdata in query: - info = str(rdata).split() - server = {'name': info[3], 'port': info[2]} - hosts[service['name']].append(server) - except DNSException: - print("Could not resolve " + - service['name'] + '.' + - group + '._tcp.' + domain) + fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain + hosts[service['name']] = do_query(fqdn) return hosts + + +# Do SRV queries +# Return array: [{"name": "f.q.d.n", "port": 8876}] +def do_query(fqdn): + servers = [] + try: + query = dns.resolver.query(fqdn, 'SRV') + query.lifetime = 1.0 + + for rdata in query: + info = str(rdata).split() + server = {'name': info[3], 'port': info[2]} + servers.append(server) + except DNSException: + print("Could not resolve " + fqdn) + + return servers From 1b962b12d40a43280de9ab4ed974d69613081bf4 Mon Sep 17 00:00:00 2001 From: w1r0x Date: Fri, 12 Aug 2016 10:39:50 +0300 Subject: [PATCH 10/39] Remove ending '.' in hostname. (for those fucking libs that knows nothing about RFC) --- surok/discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surok/discovery.py b/surok/discovery.py index 689463b..36eb77a 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -41,7 +41,7 @@ def do_query(fqdn): for rdata in query: info = str(rdata).split() - server = {'name': info[3], 'port': info[2]} + server = {'name': info[3][:-1], 'port': info[2]} servers.append(server) except DNSException: print("Could not resolve " + fqdn) From 50596fdb9f64bed3c43a9a6dbd2fd79183951488 Mon Sep 17 00:00:00 2001 From: w1r0x Date: Fri, 12 Aug 2016 10:49:19 +0300 Subject: [PATCH 11/39] Modify debian changelog --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index d20ab1b..2873832 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +surok (0.3.2) testing; urgency=medium + + * Remove ending '.' in hostname. (for those fucking libs that knows nothing about RFC) + + -- Denis Ryabyy Fri, 12 Aug 2016 10:48:12 +0300 + surok (0.1-1) testing; urgency=low * Initial release (Closes: BBONL-1696) From d6ddf8db0745612cba60fe4f4a9e719748316641 Mon Sep 17 00:00:00 2001 From: w1r0x Date: Fri, 12 Aug 2016 11:13:48 +0300 Subject: [PATCH 12/39] Remove already applyed patch from debian/patches --- debian/patches/path-change | 36 ------------------------------------ debian/patches/series | 1 - 2 files changed, 37 deletions(-) delete mode 100644 debian/patches/path-change delete mode 100644 debian/patches/series diff --git a/debian/patches/path-change b/debian/patches/path-change deleted file mode 100644 index 9687664..0000000 --- a/debian/patches/path-change +++ /dev/null @@ -1,36 +0,0 @@ -Description: - TODO: Put a short summary on the line above and replace this paragraph - with a longer explanation of this change. Complete the meta-information - with other relevant fields (see below for details). To make it easier, the - information below has been extracted from the changelog. Adjust it or drop - it. - . - surok (0.1-1) unstable; urgency=low - . - * Initial release (Closes: #nnnn) -Author: Denis Zheleztsov - ---- -The information above should follow the Patch Tagging Guidelines, please -checkout http://dep.debian.net/deps/dep3/ to learn about the format. Here -are templates for supplementary fields that you might want to add: - -Origin: , -Bug: -Bug-Debian: https://bugs.debian.org/ -Bug-Ubuntu: https://launchpad.net/bugs/ -Forwarded: -Reviewed-By: -Last-Update: - ---- surok-0.1.orig/surok.py -+++ surok-0.1/surok.py -@@ -9,7 +9,7 @@ from surok.discovery import resolve - from surok.system import reload_conf - - # Load base configurations --f = open('conf/surok.json', 'r') -+f = open('/etc/surok/conf/surok.json', 'r') - conf = json.loads(f.read()) - print(conf) - f.close() diff --git a/debian/patches/series b/debian/patches/series deleted file mode 100644 index db555fc..0000000 --- a/debian/patches/series +++ /dev/null @@ -1 +0,0 @@ -path-change From eda0acd09b17be0cbeda3ad39f106ce431959791 Mon Sep 17 00:00:00 2001 From: w1r0x Date: Fri, 12 Aug 2016 11:48:52 +0300 Subject: [PATCH 13/39] Put surok to /opt/surok --- debian/install | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debian/install b/debian/install index 52ba9b6..a8b780d 100644 --- a/debian/install +++ b/debian/install @@ -1,6 +1,6 @@ conf/surok.json etc/surok/conf -surok/templates.py usr/lib/python3/dist-packages/surok -surok/system.py usr/lib/python3/dist-packages/surok -surok/__init__.py usr/lib/python3/dist-packages/surok -surok/discovery.py usr/lib/python3/dist-packages/surok -surok.py usr/bin +surok/templates.py opt/surok/surok +surok/system.py opt/surok/surok +surok/__init__.py opt/surok/surok +surok/discovery.py opt/surok/surok +surok.py opt/surok From 97c9e04d5496bc8b0a807ca22cd42d031f17f38e Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 17 Aug 2016 12:12:49 +0300 Subject: [PATCH 14/39] New config option --- conf/surok.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/surok.json b/conf/surok.json index ecdca24..319f39b 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -3,5 +3,6 @@ "confd": "conf.d", "domain": "marathon.mesos", "wait_time": 20, - "lock_dir": "/var/tmp" + "lock_dir": "/var/tmp", + "loglevel": "info" } From da0bcda185835803f759046dce953747525fd4f7 Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 17 Aug 2016 12:13:02 +0300 Subject: [PATCH 15/39] Logging --- surok.py | 11 ++++++++--- surok/discovery.py | 8 +++++--- surok/system.py | 26 +++++++++++++++++--------- surok/templates.py | 4 ++-- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/surok.py b/surok.py index f7657b9..4bea4d7 100755 --- a/surok.py +++ b/surok.py @@ -24,7 +24,6 @@ if args.config: # Read config file f = open(surok_conf, 'r') conf = json.loads(f.read()) -print(conf) f.close() @@ -61,6 +60,13 @@ while 1: for app in confs: app_conf = load_app_conf(app) + # Will be removed later + # For old configs + try: + loglevel = conf['loglevel'] + except: + conf['loglevel'] = 'info' + # Resolve services app_hosts = resolve(app_conf, conf) @@ -71,7 +77,6 @@ while 1: # Generate config from template service_conf = gen(my, app_conf['template']) - stdout, first = reload_conf(service_conf, app_conf, first) - print(stdout) + first = reload_conf(service_conf, app_conf, first, conf) sleep(conf['wait_time']) diff --git a/surok/discovery.py b/surok/discovery.py index 36eb77a..3ad583a 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -1,6 +1,7 @@ import dns.resolver import dns.query from dns.exception import DNSException +import logging # Resolve service from mesos-dns SRV record @@ -26,14 +27,14 @@ def resolve(app, conf): group = service['group'] fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain - hosts[service['name']] = do_query(fqdn) + hosts[service['name']] = do_query(fqdn, conf['loglevel']) return hosts # Do SRV queries # Return array: [{"name": "f.q.d.n", "port": 8876}] -def do_query(fqdn): +def do_query(fqdn, loglevel): servers = [] try: query = dns.resolver.query(fqdn, 'SRV') @@ -44,6 +45,7 @@ def do_query(fqdn): server = {'name': info[3][:-1], 'port': info[2]} servers.append(server) except DNSException: - print("Could not resolve " + fqdn) + if loglevel != 'info': + logging.warning("Could not resolve " + fqdn) return servers diff --git a/surok/system.py b/surok/system.py index 87ca63c..13b4bd6 100644 --- a/surok/system.py +++ b/surok/system.py @@ -1,8 +1,10 @@ import os +import logging +# Get old configuration def get_old(name, service_conf): - + try: path = '/var/tmp/surok.' + name f = open(path, 'r') @@ -16,7 +18,7 @@ def get_old(name, service_conf): return 1 else: return 0 - + def write_lock(name, service_conf): path = '/var/tmp/surok.' + name @@ -26,7 +28,7 @@ def write_lock(name, service_conf): def do_reload(service_conf, app_conf): - print( 'Write new configuration of ' + app_conf['conf_name'] ) + logging.warning('Write new configuration of ' + app_conf['conf_name']) f = open(app_conf['dest'], 'w') f.write(service_conf) @@ -39,16 +41,22 @@ def do_reload(service_conf, app_conf): return stdout -def reload_conf(service_conf, app_conf, first): - +def reload_conf(service_conf, app_conf, first, conf): + # Check first loop - if first == True: + if first is True: stdout = do_reload(service_conf, app_conf) first = False - return stdout, first + logging.info(stdout) + return first if get_old(app_conf['conf_name'], service_conf) != 1: stdout = do_reload(service_conf, app_conf) - return stdout, first + logging.info(stdout) + return first else: - return 'Same config ' + app_conf['conf_name'] + ' Skip reload', first + if conf['loglevel'] == 'debug': + logging.debug('Same config ' + + app_conf['conf_name'] + + ' Skip reload') + return first diff --git a/surok/templates.py b/surok/templates.py index e504898..c7a1ad4 100644 --- a/surok/templates.py +++ b/surok/templates.py @@ -1,8 +1,8 @@ -from jinja2 import Environment, PackageLoader, Template +from jinja2 import Template import os -# Return rendered configuration +# Return rendered configuration def gen(my, jj2): f = open(jj2, 'r') temp = f.read() From 043682fbf6ffc39fb32fa4108d4c3cc6a2ffaf88 Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 17 Aug 2016 16:38:54 +0300 Subject: [PATCH 16/39] Fast port naming service resolve. Only one port for one service --- surok/discovery.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/surok/discovery.py b/surok/discovery.py index 3ad583a..6c1a6ad 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -12,6 +12,7 @@ def resolve(app, conf): domain = conf['domain'] group = None + # Groups hack if app['env'].get('SUROK_DISCOVERY_GROUP') is not None: group = app['env']['SUROK_DISCOVERY_GROUP'] @@ -26,7 +27,21 @@ def resolve(app, conf): # /etc/surok/conf.d/service_conf.json group = service['group'] - fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain + # Port name from app config + port_name = None + try: + port_name = service['port_name'] + except: + pass + + # This is fast fix for port naming + # Will be rewrite later + fqdn = '' + if port_name is not None: + fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain + else: + fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain + hosts[service['name']] = do_query(fqdn, conf['loglevel']) return hosts From f8a5ca2263bbd301ac7c021920ffcce7f4ea5537 Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 17 Aug 2016 16:40:40 +0300 Subject: [PATCH 17/39] changelog update --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 2873832..85128c1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +surok (0.5) UNRELEASED; urgency=medium + + * Port naming initial. Only one port for one service + + -- Denis Zheleztsov Wed, 17 Aug 2016 16:39:17 +0300 + surok (0.3.2) testing; urgency=medium * Remove ending '.' in hostname. (for those fucking libs that knows nothing about RFC) From 56f808951e8bd7850eb0da4a540a5fddb70f42c6 Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 17 Aug 2016 16:50:56 +0300 Subject: [PATCH 18/39] multiports --- surok/discovery.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/surok/discovery.py b/surok/discovery.py index 6c1a6ad..310900c 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -28,21 +28,22 @@ def resolve(app, conf): group = service['group'] # Port name from app config - port_name = None + ports = None try: - port_name = service['port_name'] + ports = service['ports'] except: pass # This is fast fix for port naming # Will be rewrite later fqdn = '' - if port_name is not None: - fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain + if ports is not None: + for port_name in ports: + fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain + hosts[service['name']]['ports'][port_name] = do_query(fqdn, conf['loglevel']) else: fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain - - hosts[service['name']] = do_query(fqdn, conf['loglevel']) + hosts[service['name']] = do_query(fqdn, conf['loglevel']) return hosts From 308397b9617ca0b2cce60dd32ebdac20c898aac5 Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 17 Aug 2016 16:51:52 +0300 Subject: [PATCH 19/39] changelog update --- debian/changelog | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 85128c1..f41599b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,9 @@ -surok (0.5) UNRELEASED; urgency=medium +surok (0.5.1) UNRELEASED; urgency=medium - * Port naming initial. Only one port for one service + * Port naming initial + * Multiports support - -- Denis Zheleztsov Wed, 17 Aug 2016 16:39:17 +0300 + -- Denis Zheleztsov Wed, 17 Aug 2016 16:50:58 +0300 surok (0.3.2) testing; urgency=medium From b390210173cb14cf8cd31f1d92e18979d9b49c3a Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 17 Aug 2016 17:16:43 +0300 Subject: [PATCH 20/39] Fix ports discovery --- surok/discovery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surok/discovery.py b/surok/discovery.py index 310900c..5e5ad05 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -17,7 +17,7 @@ def resolve(app, conf): group = app['env']['SUROK_DISCOVERY_GROUP'] for service in services: - hosts[service['name']] = [] + hosts[service['name']] = {} # Check group configuration if group is not None: @@ -40,7 +40,7 @@ def resolve(app, conf): if ports is not None: for port_name in ports: fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain - hosts[service['name']]['ports'][port_name] = do_query(fqdn, conf['loglevel']) + hosts[service['name']][port_name] = do_query(fqdn, conf['loglevel']) else: fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain hosts[service['name']] = do_query(fqdn, conf['loglevel']) From e3b7fffe56040ddaf487814c466ca0168e43fcfe Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 17 Aug 2016 17:18:12 +0300 Subject: [PATCH 21/39] changelog update --- debian/changelog | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index f41599b..18fa37f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,8 @@ -surok (0.5.1) UNRELEASED; urgency=medium +surok (0.5.2) UNRELEASED; urgency=medium - * Port naming initial - * Multiports support + * Fix ports discovery - -- Denis Zheleztsov Wed, 17 Aug 2016 16:50:58 +0300 + -- Denis Zheleztsov Wed, 17 Aug 2016 17:17:18 +0300 surok (0.3.2) testing; urgency=medium From f54a7e0ee87a72efc90fc0e248465436bbb7f5f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A4=D0=B5=D0=B4=D0=BE=D1=80=D1=87=D1=83=D0=BA=20=D0=92?= =?UTF-8?q?=D0=BB=D0=B0=D0=B4=D0=B8=D0=BC=D0=B8=D1=80=20=D0=9D=D0=B8=D0=BA?= =?UTF-8?q?=D0=BE=D0=BB=D0=B0=D0=B5=D0=B2=D0=B8=D1=87?= Date: Tue, 20 Sep 2016 14:40:34 +0300 Subject: [PATCH 22/39] Update surok.json --- conf/surok.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/surok.json b/conf/surok.json index 319f39b..86ab76b 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,6 +1,6 @@ { "marathon": "TODO", - "confd": "conf.d", + "confd": "/etc/surok/conf.d", "domain": "marathon.mesos", "wait_time": 20, "lock_dir": "/var/tmp", From f98677deb932dcb09268e73ada43e932e1922886 Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 5 Oct 2016 11:46:51 +0300 Subject: [PATCH 23/39] fix reload in empty discovery --- surok.py | 15 ++++++++------- surok/discovery.py | 25 ++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/surok.py b/surok.py index 4bea4d7..124d58e 100755 --- a/surok.py +++ b/surok.py @@ -68,15 +68,16 @@ while 1: conf['loglevel'] = 'info' # Resolve services - app_hosts = resolve(app_conf, conf) + if resolve(app_conf, conf) != 404: + app_hosts = resolve(app_conf, conf) - # Populate my dictionary - my = {"services": app_hosts, - "conf_name": app_conf['conf_name']} + # Populate my dictionary + my = {"services": app_hosts, + "conf_name": app_conf['conf_name']} - # Generate config from template - service_conf = gen(my, app_conf['template']) + # Generate config from template + service_conf = gen(my, app_conf['template']) - first = reload_conf(service_conf, app_conf, first, conf) + first = reload_conf(service_conf, app_conf, first, conf) sleep(conf['wait_time']) diff --git a/surok/discovery.py b/surok/discovery.py index 5e5ad05..e60d873 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -40,8 +40,30 @@ def resolve(app, conf): if ports is not None: for port_name in ports: fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain + + # Fast fix of empty query result + # ------------------------------ + try: + do_query_test = do_query(fqdn, conf['loglevel']) + if do_query_test['state'] == 404: + return 404 + except: + pass + # ------------------------------ + hosts[service['name']][port_name] = do_query(fqdn, conf['loglevel']) else: + + # Fast fix of empty query result + # ------------------------------ + try: + do_query_test = do_query(fqdn, conf['loglevel']) + if do_query_test['state'] == 404: + return 404 + except: + pass + # ------------------------------ + fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain hosts[service['name']] = do_query(fqdn, conf['loglevel']) @@ -62,6 +84,7 @@ def do_query(fqdn, loglevel): servers.append(server) except DNSException: if loglevel != 'info': - logging.warning("Could not resolve " + fqdn) + logging.error("Could not resolve " + fqdn) + return {"state": 404} return servers From af776539b1bb50f67a4e6082f5d626ac285f8ccc Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 5 Oct 2016 13:14:12 +0300 Subject: [PATCH 24/39] more informative error log --- surok/discovery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/surok/discovery.py b/surok/discovery.py index e60d873..5b8322a 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -82,9 +82,9 @@ def do_query(fqdn, loglevel): info = str(rdata).split() server = {'name': info[3][:-1], 'port': info[2]} servers.append(server) - except DNSException: + except DNSException as e: if loglevel != 'info': - logging.error("Could not resolve " + fqdn) + logging.error("Could not resolve " + fqdn + ': ' + str(e)) return {"state": 404} return servers From b1d0333eb7292e7884c0fcd1d4afdb459b648c22 Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 5 Oct 2016 14:11:25 +0300 Subject: [PATCH 25/39] Fix year in License --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 17ad9a0..ee0249a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2106, Denis Zheleztsov +Copyright (c) 2016, Denis Zheleztsov All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: From 4169660ae5014be82ba16343aaad689418bd29a7 Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 5 Oct 2016 14:16:51 +0300 Subject: [PATCH 26/39] log format --- surok/discovery.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/surok/discovery.py b/surok/discovery.py index 5b8322a..e3d7cb1 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -4,6 +4,11 @@ from dns.exception import DNSException import logging +# Configure logging +FORMAT = '%(asctime) %(message)s' +logging.basicConfig(format=FORMAT) +logger = logging.getLogger('discovery') + # Resolve service from mesos-dns SRV record # return dict {"servicename": [{"name": "service.f.q.d.n.", "port": 9999}]} def resolve(app, conf): @@ -84,7 +89,7 @@ def do_query(fqdn, loglevel): servers.append(server) except DNSException as e: if loglevel != 'info': - logging.error("Could not resolve " + fqdn + ': ' + str(e)) + logger.error("Could not resolve " + fqdn + ': ' + str(e)) return {"state": 404} return servers From c04242a47fc01c01c0daa609b1816d468daa1a05 Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 11 Oct 2016 12:14:34 +0300 Subject: [PATCH 27/39] Groups switch support #SD-10 --- surok/discovery.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/surok/discovery.py b/surok/discovery.py index 5e5ad05..737dc9f 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -65,3 +65,9 @@ def do_query(fqdn, loglevel): logging.warning("Could not resolve " + fqdn) return servers + + +# Groups switch +# Priority: config, environment, marathon environment +def get_group(conf): + pass From f0a509049cf189bc974943b774c0387e9f581a59 Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 11 Oct 2016 12:17:58 +0300 Subject: [PATCH 28/39] Revert stupid resolve code --- surok.py | 15 +++++++-------- surok/discovery.py | 24 +----------------------- 2 files changed, 8 insertions(+), 31 deletions(-) diff --git a/surok.py b/surok.py index 124d58e..4bea4d7 100755 --- a/surok.py +++ b/surok.py @@ -68,16 +68,15 @@ while 1: conf['loglevel'] = 'info' # Resolve services - if resolve(app_conf, conf) != 404: - app_hosts = resolve(app_conf, conf) + app_hosts = resolve(app_conf, conf) - # Populate my dictionary - my = {"services": app_hosts, - "conf_name": app_conf['conf_name']} + # Populate my dictionary + my = {"services": app_hosts, + "conf_name": app_conf['conf_name']} - # Generate config from template - service_conf = gen(my, app_conf['template']) + # Generate config from template + service_conf = gen(my, app_conf['template']) - first = reload_conf(service_conf, app_conf, first, conf) + first = reload_conf(service_conf, app_conf, first, conf) sleep(conf['wait_time']) diff --git a/surok/discovery.py b/surok/discovery.py index 218465b..8042a1f 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -45,30 +45,8 @@ def resolve(app, conf): if ports is not None: for port_name in ports: fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain - - # Fast fix of empty query result - # ------------------------------ - try: - do_query_test = do_query(fqdn, conf['loglevel']) - if do_query_test['state'] == 404: - return 404 - except: - pass - # ------------------------------ - hosts[service['name']][port_name] = do_query(fqdn, conf['loglevel']) - else: - - # Fast fix of empty query result - # ------------------------------ - try: - do_query_test = do_query(fqdn, conf['loglevel']) - if do_query_test['state'] == 404: - return 404 - except: - pass - # ------------------------------ - + else: fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain hosts[service['name']] = do_query(fqdn, conf['loglevel']) From e57f08edbbd91dff7d03bc42f67135dc83a3cb50 Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 11 Oct 2016 14:36:41 +0300 Subject: [PATCH 29/39] Closed SD-10 --- surok/discovery.py | 65 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/surok/discovery.py b/surok/discovery.py index 8042a1f..604c441 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -2,12 +2,18 @@ import dns.resolver import dns.query from dns.exception import DNSException import logging +import sys -# Configure logging -FORMAT = '%(asctime) %(message)s' -logging.basicConfig(format=FORMAT) -logger = logging.getLogger('discovery') +# Logger configuration +# This need to be moved +def get_logger(): + # Configure logging + FORMAT = '%(asctime) %(message)s' + logging.basicConfig(format=FORMAT) + logger = logging.getLogger(__name__) + return logger + # Resolve service from mesos-dns SRV record # return dict {"servicename": [{"name": "service.f.q.d.n.", "port": 9999}]} @@ -15,23 +21,17 @@ def resolve(app, conf): hosts = {} services = app['services'] domain = conf['domain'] - group = None - - # Groups hack - if app['env'].get('SUROK_DISCOVERY_GROUP') is not None: - group = app['env']['SUROK_DISCOVERY_GROUP'] + logger = get_logger() for service in services: hosts[service['name']] = {} - # Check group configuration - if group is not None: - pass - else: - # Load group from service config - # /etc/surok/conf.d/service_conf.json - group = service['group'] - + group = get_group(service, app) + if group is False: + logger.error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID') + logger.error('Not in Mesos launch?') + sys.exit(2) + # Port name from app config ports = None try: @@ -56,6 +56,7 @@ def resolve(app, conf): # Do SRV queries # Return array: [{"name": "f.q.d.n", "port": 8876}] def do_query(fqdn, loglevel): + logger = get_logger() servers = [] try: query = dns.resolver.query(fqdn, 'SRV') @@ -75,5 +76,31 @@ def do_query(fqdn, loglevel): # Groups switch # Priority: config, environment, marathon environment -def get_group(conf): - pass +def get_group(service, app): + # Check group in app conf + if 'group' in service: + return service['group'] + # Check environment variable + elif app['env'].get('SUROK_DISCOVERY_GROUP'): + return app['env']['SUROK_DISCOVERY_GROUP'] + # Check marathon environment variable + elif app['env'].get('MARATHON_APP_ID'): + group = parse_marathon_app_id(app['env']['MARATHON_APP_ID']) + return group + else: + return False + + +# Parse MARATHON_APP_ID +def parse_marathon_app_id(marathon_app_id): + marathon_app_id = marathon_app_id.split('/') + group = '' + counter = len(marathon_app_id) - 2 + i = 0 + while counter > i: + group = group + marathon_app_id[counter] + if counter != i + 1: + group += '.' + counter -= 1 + + return group From f1746a0ed6e9e4472855268ca489d580bfc6b03c Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 12 Oct 2016 16:03:18 +0300 Subject: [PATCH 30/39] Changelog update --- debian/changelog | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 18fa37f..2755750 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,10 @@ -surok (0.5.2) UNRELEASED; urgency=medium +surok (0.5.5) testing; urgency=medium - * Fix ports discovery + * #closes SD-10 + * Group switch + * Version bump - -- Denis Zheleztsov Wed, 17 Aug 2016 17:17:18 +0300 + -- Denis Zheleztsov Tue, 11 Oct 2016 15:14:44 +0300 surok (0.3.2) testing; urgency=medium From 77451203441dd4e9243ec53f356e5222fe36ee5e Mon Sep 17 00:00:00 2001 From: Difrex Date: Thu, 13 Oct 2016 13:12:26 +0300 Subject: [PATCH 31/39] Readme --- README.en.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 README.en.md diff --git a/README.en.md b/README.en.md new file mode 100644 index 0000000..29fdcfa --- /dev/null +++ b/README.en.md @@ -0,0 +1,8 @@ +# Surok + +Service discovery for Apache Mesos. + +* Jinja2 Templates +* Discovery over mesos-dns +* Applications config reload + From 0d881c9008847c51a10500111a9c0bf6382d9f84 Mon Sep 17 00:00:00 2001 From: Difrex Date: Thu, 13 Oct 2016 13:25:17 +0300 Subject: [PATCH 32/39] Ru doc initial --- doc/ru/app.configuration.md | 33 +++++++++++ doc/ru/surok.configuration.md | 23 ++++++++ doc/ru/templates.md | 104 ++++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 doc/ru/app.configuration.md create mode 100644 doc/ru/surok.configuration.md create mode 100644 doc/ru/templates.md diff --git a/doc/ru/app.configuration.md b/doc/ru/app.configuration.md new file mode 100644 index 0000000..f002724 --- /dev/null +++ b/doc/ru/app.configuration.md @@ -0,0 +1,33 @@ +# Конфигурация приложения + +/etc/surok/conf.d/app.json +``` +{ + "services": [ + { + "name": "kioskservice", + "group": "production.romania", + "ports": ["web", "socket"] + } + ], + "conf_name": "kiosk", + "template": "/etc/surok/templates/kiosk.jj2", + "dest": "/etc/nginx/sites-available/kioskservice.conf", + "reload_cmd": "/bin/systemctl reload nginx", + "run_cmd": ["/usr/bin/node", "-c", "config.json"] +} +``` + +Давайте разберем конфигурационный файл по опциям +* services - array. Список хэшей с описанием сервисов + name - string. Имя сервиса. Это имя приложения в marathon + group - string. Группа в которой находится сервис. Группу можно узнать в marathon. Записывается в обратном порядке. Т.е. если у нас есть группа /webapps/php, то записывать её следует, как php.webapps + Если группа не указана, то сурок ожидает группу в переменной окружения SUROK_DISCOVERY_GROUP, если и SUROK_DISCOVERY_GROUP нет, то берется группа marathon(0.5.5). + ports - array. Список имен портов сервиса. Не обязательная опция. +* conf_name - string. Название конфига. Должен быть уникальным значением. Слежит для создания и чтения lock конфигурации. +* template - string. Абсолютный путь к файлу шаблона. +* dest - string. Абсолютный путь к файлу в который запишется результат генерации шаблона. +* reload_cmd - string. Команда, которая будет выполнена в случае обноления конфига. + В reload_cmd можно использовать переменные окружения: + ```"reload_cmd": "/usr/bin/killall -9 calc || true && /usr/local/bin/calc -c /app/calc.conf ${CALC_NUM}"``` +* run_cmd(v0.6) - array. Список с командой на выполнение. Используется внутри контейнера вместо reload_cmd. diff --git a/doc/ru/surok.configuration.md b/doc/ru/surok.configuration.md new file mode 100644 index 0000000..9b9a19f --- /dev/null +++ b/doc/ru/surok.configuration.md @@ -0,0 +1,23 @@ +# Конфигурация Surok + +**/etc/surok/conf/surok.json** +Разберем конфигурационный файл по опциям +``` +{ + "marathon": "10.0.1.199:8080", + "confd": "/etc/surok/conf.d", + "domain": "marathon.mesos", + "wait_time": 20, + "lock_dir": "/var/tmp", + "loglevel": "info|debug" + "container": true|false +} +``` + +* marathon(v0.7) - string. Адрес Marathon Sheduler. +* confd - strig. Абсолютный путь до директории с конфигурационными файлами приложений. +* domain - string. Домен, который обслуживает mesos-dns. +* wait_time - int. Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов. +* lock_dir - string. Абсолютный путь до директории с lock-конфигурациями. +* loglevel - string. Уровень логирования. +* container(v0.6) - boolean. Определяем внутри или нет контейнера запущен сурок. Меняется логика работы. diff --git a/doc/ru/templates.md b/doc/ru/templates.md new file mode 100644 index 0000000..5ac75c3 --- /dev/null +++ b/doc/ru/templates.md @@ -0,0 +1,104 @@ +# Шиблоны + +Шаблоны для Surok пишутся на Jinja2. Возможно, стоит прочитать документацию. + +## Словарь my в шаблоне + +Surok заполняет словарь my и передает его в шаблон. +``` +{ + "services": { + "nginx": [ + { + "name": "nginx.testing-kl92-s0.marathon.mesos.", + "port": "31200" + }, + { + "name": "nginx.testing-kl123-s1.marathon.mesos.", + "port": "32230" + } + ], + "emailsender": [ + { + "name": "emailsender.testing-kl92-s0.marathon.mesos.", + "port": "31201" + }, + { + "name": "emailsender.testing-kl123-s1.marathon.mesos.", + "port": "32232" + } + ], + "service-with-defined-ports": { + "web": [ + { + "name": "f.q.d.n", + "port": 12341 + } + ], + "rpc": [ + { + "name": "f.q.d.n", + "port": 12342 + } + ] + } + }, + "env": { + "HOME": "/var/lib/nginx" + } +} +``` + +## Пример реального шаблона + +``` +upstream matrix-http { + hash $remote_addr; +{% for server in my['services']['matrix']['http'] %} + server {{server['name']}}:{{server['port']}} max_fails=3; +{% endfor %} +} + +upstream riot-http { + hash $remote_addr; +{% for server in my['services']['riot'] %} + server {{server['name']}}:{{server['port']}} max_fails=3; +{% endfor %} +} + +server { + listen 10.15.56.157:80; + server_name matrix.example.com; + + client_max_body_size 10m; + + location / { + proxy_pass http://riot-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /_matrix/ { + proxy_pass http://matrix-http; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + +} +``` +Так для upstream matrix-http используются именованные порты, а для riot-http – нет. + +## Проверки в шаблоне + +Переменная _my['env']_ является классом python _os.environ_, что позваоляет нам строить различные проверки, например: + +``` +{% if my['env'].get('DB_HOST') %} +host = '{{my['env']['DB_HOST']}}' +{% else %} +host = 'localhost' +{% endif %} +``` + From 73cf76cd861d62fce84a946800205a3aef4096bc Mon Sep 17 00:00:00 2001 From: Difrex Date: Fri, 14 Oct 2016 11:01:52 +0300 Subject: [PATCH 33/39] marathon config --- conf/surok.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/surok.json b/conf/surok.json index 86ab76b..6fc5196 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,5 +1,8 @@ { - "marathon": "TODO", + "marathon": { + "force": true, + "host": "marathon.mesos:8080" + }, "confd": "/etc/surok/conf.d", "domain": "marathon.mesos", "wait_time": 20, From d15a23774c9b4c050f4ac6618f77022e31003910 Mon Sep 17 00:00:00 2001 From: Difrex Date: Fri, 14 Oct 2016 11:25:53 +0300 Subject: [PATCH 34/39] Marathon restart implementation --- conf/surok.json | 3 ++- surok/discovery.py | 1 + surok/system.py | 25 +++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/conf/surok.json b/conf/surok.json index 6fc5196..427a982 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,7 +1,8 @@ { "marathon": { "force": true, - "host": "marathon.mesos:8080" + "host": "http://marathon.mesos:8080", + "enabled": true }, "confd": "/etc/surok/conf.d", "domain": "marathon.mesos", diff --git a/surok/discovery.py b/surok/discovery.py index 604c441..11298de 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -92,6 +92,7 @@ def get_group(service, app): # Parse MARATHON_APP_ID +# Return marathon.group def parse_marathon_app_id(marathon_app_id): marathon_app_id = marathon_app_id.split('/') group = '' diff --git a/surok/system.py b/surok/system.py index 13b4bd6..8a1914b 100644 --- a/surok/system.py +++ b/surok/system.py @@ -1,5 +1,6 @@ import os import logging +import requests # Get old configuration @@ -50,6 +51,10 @@ def reload_conf(service_conf, app_conf, first, conf): logging.info(stdout) return first + # Check marathon enabled in configuration + if conf['marathon']['enabled'] is True: + restart_self_in_marathon(conf['marathon']) + if get_old(app_conf['conf_name'], service_conf) != 1: stdout = do_reload(service_conf, app_conf) logging.info(stdout) @@ -60,3 +65,23 @@ def reload_conf(service_conf, app_conf, first, conf): app_conf['conf_name'] + ' Skip reload') return first + + +# Do POST request to marathon API +# /v2/apps//app/name/restart +def restart_self_in_marathon(marathon): + host = marathon['host'] + + # Check MARATHON_APP_ID environment varible + if os.environ.get('MARATHON_APP_ID') is not True: + logging.error('Cannot find MARATHON_APP_ID. Not in Mesos?') + sys.exit(2) + app_id = os.environ['MARATHON_APP_ID'] + uri = 'http://' + host + '/v2/apps/' + app_id + '/restart' + + # Ok. In this step we made restart request to Marathon + if marathon['force'] is True: + r = requests.post(uri, data = {'force': 'true'}) + else: + r = requests.post(uri, data = {}) + From 013c7dfbe91dc61cb2a451250498dae4e9892ae9 Mon Sep 17 00:00:00 2001 From: Difrex Date: Fri, 14 Oct 2016 11:50:16 +0300 Subject: [PATCH 35/39] fixed sys module --- surok/system.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surok/system.py b/surok/system.py index 8a1914b..8d7ecff 100644 --- a/surok/system.py +++ b/surok/system.py @@ -1,4 +1,5 @@ import os +import sys import logging import requests From e70d24a4680ecae8ae9c137cc757099494327d92 Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Fri, 14 Oct 2016 11:53:31 +0300 Subject: [PATCH 36/39] changelog and control update --- debian/changelog | 14 ++++++++++++++ debian/control | 2 +- debian/source/format | 1 - 3 files changed, 15 insertions(+), 2 deletions(-) delete mode 100644 debian/source/format diff --git a/debian/changelog b/debian/changelog index 2755750..a750712 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,17 @@ +surok (0.7-1) testing; urgency=medium + + * New dependencies + + -- Denis Zheleztsov Fri, 14 Oct 2016 11:39:36 +0300 + +surok (0.7) testing; urgency=medium + + * Marathon restart implementation + * WARNING: BROKEN BACKWARD COMPATIBILY WITH OLD MAIN CONFIG + !!! Please update your config first !!! + + -- Denis Zheleztsov Fri, 14 Oct 2016 11:28:07 +0300 + surok (0.5.5) testing; urgency=medium * #closes SD-10 diff --git a/debian/control b/debian/control index 2be1fa5..9af93c8 100644 --- a/debian/control +++ b/debian/control @@ -8,5 +8,5 @@ Vcs-Git: http://vcs.bingo-boom.ru/difrex/surok.git Package: surok Architecture: all -Depends: python3-jinja2, python3-dnsq +Depends: python3-jinja2, python3-dnsq, python3-requests Description: Service discovery for Apache Mesos clusters diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 163aaf8..0000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (quilt) From 07c2b3e3e485da2634cfca57bd97468d7a476f30 Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Fri, 14 Oct 2016 12:41:19 +0300 Subject: [PATCH 37/39] Small fix --- surok/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surok/system.py b/surok/system.py index 8d7ecff..ec3470a 100644 --- a/surok/system.py +++ b/surok/system.py @@ -84,5 +84,5 @@ def restart_self_in_marathon(marathon): if marathon['force'] is True: r = requests.post(uri, data = {'force': 'true'}) else: - r = requests.post(uri, data = {}) + r = requests.post(uri, data = {'force': 'false'}) From 79559a966f5a333962f707b3fa7922c3b4594700 Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Fri, 14 Oct 2016 12:42:04 +0300 Subject: [PATCH 38/39] changelog update --- debian/changelog | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/debian/changelog b/debian/changelog index a750712..09b2806 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +surok (0.7-3) jessie; urgency=medium + + * Small fix in marathon restart + + -- Denis Zheleztsov Fri, 14 Oct 2016 12:41:31 +0300 + +surok (0.7-2) testing; urgency=medium + + * Fixed sys module + + -- Denis Zheleztsov Fri, 14 Oct 2016 11:53:45 +0300 + surok (0.7-1) testing; urgency=medium * New dependencies From f832f5bc8792f125f4595e42eee4100fe982ff59 Mon Sep 17 00:00:00 2001 From: Difrex Date: Fri, 14 Oct 2016 14:12:27 +0300 Subject: [PATCH 39/39] revert 404 --- surok/discovery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/surok/discovery.py b/surok/discovery.py index 11298de..72a7887 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -69,7 +69,6 @@ def do_query(fqdn, loglevel): except DNSException as e: if loglevel != 'info': logger.error("Could not resolve " + fqdn + ': ' + str(e)) - return {"state": 404} return servers