From d62a424cf2a350b6ba01ec6cc4a059bfabdca73d Mon Sep 17 00:00:00 2001 From: Difrex Date: Tue, 2 Aug 2016 19:02:25 +0300 Subject: [PATCH 1/4] Multiple service discovery --- .gitignore | 2 ++ conf.d/1.json | 13 ++++++++++++- conf/surok.json | 5 ++++- surok.py | 4 ++-- surok/discovery.py | 10 ++++++---- surok/system.py | 4 ++-- templates/nginx_proxy.jj2 | 11 +++++++++++ 7 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 templates/nginx_proxy.jj2 diff --git a/.gitignore b/.gitignore index ffa90d4..6e332f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ surok/__pycache__ __pycache__ +*.pyc +*.swp diff --git a/conf.d/1.json b/conf.d/1.json index 0967ef4..152ae26 100644 --- a/conf.d/1.json +++ b/conf.d/1.json @@ -1 +1,12 @@ -{} +{ + "services": [ + { + "name": "kioskservice", + "group": "production.romania" + } + ], + "conf_name": "test", + "template": "templates/nginx_proxy.jj2", + "dest": "emailsender", + "reload_cmd": "/bin/echo reload nginx" +} diff --git a/conf/surok.json b/conf/surok.json index 96046a7..58e5694 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,4 +1,7 @@ { "marathon": "10.0.1.199:8080", - "confd": "conf.d" + "confd": "conf.d", + "domain": "marathon.mesos", + "wait_time": 20, + "lock_dir": "/var/tmp" } diff --git a/surok.py b/surok.py index 1417e3a..1cf77e8 100755 --- a/surok.py +++ b/surok.py @@ -9,7 +9,7 @@ from surok.discovery import resolve from surok.system import reload_conf # Load base configurations -f = open('/etc/surok/conf/surok.json', 'r') +f = open('conf/surok.json', 'r') conf = json.loads(f.read()) print(conf) f.close() @@ -36,7 +36,7 @@ while 1: for app in confs: app_conf = load_app_conf(app) app_hosts = resolve(app_conf, conf) - my = {'app': app_conf['name'], 'hosts': app_hosts} + my = { 'hosts': app_hosts } service_conf = gen(my, app_conf['template']) print(reload_conf(service_conf, app_conf)) diff --git a/surok/discovery.py b/surok/discovery.py index b43a298..6d8297a 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -2,10 +2,12 @@ import dns.resolver def resolve(app, conf): hosts = [] + services = app['services'] domain = conf['domain'] - for rdata in dns.resolver.query('_' + app['name'] + '.' + app['group'] + '._tcp.' + domain, 'SRV'): - info = str(rdata).split() - server = { 'name': info[3], 'port': info[2] } - hosts.append(server) + for service in services: + for rdata in dns.resolver.query('_' + service['name'] + '.' + service['group'] + '._tcp.' + domain, 'SRV'): + info = str(rdata).split() + server = { service['name']: { 'name': info[3], 'port': info[2] } } + hosts.append(server) return hosts diff --git a/surok/system.py b/surok/system.py index 686fca8..bd1bb7e 100644 --- a/surok/system.py +++ b/surok/system.py @@ -28,14 +28,14 @@ def write_lock(name, service_conf): def reload_conf(service_conf, app_conf): # Check old config - if get_old(app_conf['name'], service_conf) != 1: + if get_old(app_conf['conf_name'], service_conf) != 1: print('Write new configuration') f = open(app_conf['dest'], 'w') f.write(service_conf) f.close() - write_lock(app_conf['name'], service_conf) + write_lock(app_conf['conf_name'], service_conf) # Reload conf stdout = os.popen(app_conf['reload_cmd']).read() diff --git a/templates/nginx_proxy.jj2 b/templates/nginx_proxy.jj2 new file mode 100644 index 0000000..4abe0b3 --- /dev/null +++ b/templates/nginx_proxy.jj2 @@ -0,0 +1,11 @@ + {% for host in my['hosts'] %} + server {{host['kioskservice']['name']}}:{{host['kioskservice']['port']}}; + {% endfor %} + +server { + listen 0.0.0.0:80; + server_name _; + + location / { + } +} From 0c3c2014cc52d819a7c82b8e26298daf9af2b606 Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 3 Aug 2016 09:54:09 +0300 Subject: [PATCH 2/4] my dict update --- surok.py | 2 +- surok/discovery.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/surok.py b/surok.py index 1cf77e8..e74276c 100755 --- a/surok.py +++ b/surok.py @@ -36,7 +36,7 @@ while 1: for app in confs: app_conf = load_app_conf(app) app_hosts = resolve(app_conf, conf) - my = { 'hosts': app_hosts } + my = { 'services': app_hosts } service_conf = gen(my, app_conf['template']) print(reload_conf(service_conf, app_conf)) diff --git a/surok/discovery.py b/surok/discovery.py index 6d8297a..f4a6ff4 100644 --- a/surok/discovery.py +++ b/surok/discovery.py @@ -1,13 +1,18 @@ import dns.resolver def resolve(app, conf): - hosts = [] + hosts = {} services = app['services'] domain = conf['domain'] for service in services: - for rdata in dns.resolver.query('_' + service['name'] + '.' + service['group'] + '._tcp.' + domain, 'SRV'): - info = str(rdata).split() - server = { service['name']: { 'name': info[3], 'port': info[2] } } - hosts.append(server) + hosts[service['name']] = [] + try: + for rdata in dns.resolver.query('_' + service['name'] + '.' + service['group'] + '._tcp.' + domain, 'SRV'): + info = str(rdata).split() + server = { 'name': info[3], 'port': info[2] } + + hosts[ service['name'] ].append(server) + except Exception as e: + print("Could not resolve " + service['name'] + '.' + service['group'] + '._tcp.' + domain) return hosts From 7610fc315281b229a1e5e31d404f49031054f9fa Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 3 Aug 2016 15:46:08 +0300 Subject: [PATCH 3/4] OS environment to my dict --- surok/templates.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surok/templates.py b/surok/templates.py index 6fd3363..8056af0 100644 --- a/surok/templates.py +++ b/surok/templates.py @@ -1,4 +1,5 @@ from jinja2 import Environment, PackageLoader, Template +import os # Return rendered configuration @@ -8,5 +9,6 @@ def gen(my, jj2): f.close() template = Template(temp) + my['env'] = os.environ return template.render(my=my) From 2d7f6ae8568c7ab53507de3bdaa6a0e631b1bfdc Mon Sep 17 00:00:00 2001 From: Difrex Date: Wed, 3 Aug 2016 17:26:57 +0300 Subject: [PATCH 4/4] Working configs with selfcheck --- conf.d/1.json | 12 ------------ conf.d/selfcheck.json | 12 ++++++++++++ conf/surok.json | 2 +- templates/selfcheck.jj2 | 1 + 4 files changed, 14 insertions(+), 13 deletions(-) delete mode 100644 conf.d/1.json create mode 100644 conf.d/selfcheck.json create mode 100644 templates/selfcheck.jj2 diff --git a/conf.d/1.json b/conf.d/1.json deleted file mode 100644 index 152ae26..0000000 --- a/conf.d/1.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "services": [ - { - "name": "kioskservice", - "group": "production.romania" - } - ], - "conf_name": "test", - "template": "templates/nginx_proxy.jj2", - "dest": "emailsender", - "reload_cmd": "/bin/echo reload nginx" -} diff --git a/conf.d/selfcheck.json b/conf.d/selfcheck.json new file mode 100644 index 0000000..56cec93 --- /dev/null +++ b/conf.d/selfcheck.json @@ -0,0 +1,12 @@ +{ + "services": [ + { + "name": "", + "group": "mesos" + } + ], + "conf_name": "selfcheck", + "template": "templates/selfcheck.jj2", + "dest": "selfcheck", + "reload_cmd": "/bin/echo selfcheck ok" +} diff --git a/conf/surok.json b/conf/surok.json index 58e5694..ecdca24 100644 --- a/conf/surok.json +++ b/conf/surok.json @@ -1,5 +1,5 @@ { - "marathon": "10.0.1.199:8080", + "marathon": "TODO", "confd": "conf.d", "domain": "marathon.mesos", "wait_time": 20, diff --git a/templates/selfcheck.jj2 b/templates/selfcheck.jj2 new file mode 100644 index 0000000..d86bac9 --- /dev/null +++ b/templates/selfcheck.jj2 @@ -0,0 +1 @@ +OK