Merge branch 'master' of vcs.bingo-boom.ru:difrex/surok into test

This commit is contained in:
Difrex 2016-10-18 09:21:27 +03:00
commit 49e264ddd1
17 changed files with 389 additions and 84 deletions

1
.gitignore vendored
View File

@ -2,3 +2,4 @@ surok/__pycache__
__pycache__
*.pyc
*.swp
selfcheck

View File

@ -1,4 +1,4 @@
Copyright (c) 2106, Denis Zheleztsov <difrex.punk@gmail.com>
Copyright (c) 2016, Denis Zheleztsov <difrex.punk@gmail.com>
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

8
README.en.md Normal file
View File

@ -0,0 +1,8 @@
# Surok
Service discovery for Apache Mesos.
* Jinja2 Templates
* Discovery over mesos-dns
* Applications config reload

View File

@ -1,7 +1,12 @@
{
"marathon": "TODO",
"confd": "conf.d",
"marathon": {
"force": true,
"host": "http://marathon.mesos:8080",
"enabled": true
},
"confd": "/etc/surok/conf.d",
"domain": "marathon.mesos",
"wait_time": 20,
"lock_dir": "/var/tmp"
"lock_dir": "/var/tmp",
"loglevel": "info"
}

40
debian/changelog vendored
View File

@ -1,3 +1,43 @@
surok (0.7-3) jessie; urgency=medium
* Small fix in marathon restart
-- Denis Zheleztsov <difrex@bingo-boom.ru> Fri, 14 Oct 2016 12:41:31 +0300
surok (0.7-2) testing; urgency=medium
* Fixed sys module
-- Denis Zheleztsov <difrex@bingo-boom.ru> Fri, 14 Oct 2016 11:53:45 +0300
surok (0.7-1) testing; urgency=medium
* New dependencies
-- Denis Zheleztsov <difrex@bingo-boom.ru> Fri, 14 Oct 2016 11:39:36 +0300
surok (0.7) testing; urgency=medium
* Marathon restart implementation
* WARNING: BROKEN BACKWARD COMPATIBILY WITH OLD MAIN CONFIG
!!! Please update your config first !!!
-- Denis Zheleztsov <difrex@bingo-boom.ru> Fri, 14 Oct 2016 11:28:07 +0300
surok (0.5.5) testing; urgency=medium
* #closes SD-10
* Group switch
* Version bump
-- Denis Zheleztsov <difrex@bingo-boom.ru> Tue, 11 Oct 2016 15:14:44 +0300
surok (0.3.2) testing; urgency=medium
* Remove ending '.' in hostname. (for those fucking libs that knows nothing about RFC)
-- Denis Ryabyy <vv1r0x@gmail.com> Fri, 12 Aug 2016 10:48:12 +0300
surok (0.1-1) testing; urgency=low
* Initial release (Closes: BBONL-1696)

2
debian/control vendored
View File

@ -8,5 +8,5 @@ Vcs-Git: http://vcs.bingo-boom.ru/difrex/surok.git
Package: surok
Architecture: all
Depends: python3-jinja2, python3-dnsq
Depends: python3-jinja2, python3-dnsq, python3-requests
Description: Service discovery for Apache Mesos clusters

10
debian/install vendored
View File

@ -1,6 +1,6 @@
conf/surok.json etc/surok/conf
surok/templates.py usr/lib/python3/dist-packages/surok
surok/system.py usr/lib/python3/dist-packages/surok
surok/__init__.py usr/lib/python3/dist-packages/surok
surok/discovery.py usr/lib/python3/dist-packages/surok
surok.py usr/bin
surok/templates.py opt/surok/surok
surok/system.py opt/surok/surok
surok/__init__.py opt/surok/surok
surok/discovery.py opt/surok/surok
surok.py opt/surok

View File

@ -1,36 +0,0 @@
Description: <short summary of the patch>
TODO: Put a short summary on the line above and replace this paragraph
with a longer explanation of this change. Complete the meta-information
with other relevant fields (see below for details). To make it easier, the
information below has been extracted from the changelog. Adjust it or drop
it.
.
surok (0.1-1) unstable; urgency=low
.
* Initial release (Closes: #nnnn) <nnnn is the bug number of your ITP>
Author: Denis Zheleztsov <difrex@bingo-boom.ru>
---
The information above should follow the Patch Tagging Guidelines, please
checkout http://dep.debian.net/deps/dep3/ to learn about the format. Here
are templates for supplementary fields that you might want to add:
Origin: <vendor|upstream|other>, <url of original patch>
Bug: <url in upstream bugtracker>
Bug-Debian: https://bugs.debian.org/<bugnumber>
Bug-Ubuntu: https://launchpad.net/bugs/<bugnumber>
Forwarded: <no|not-needed|url proving that it has been forwarded>
Reviewed-By: <name and email of someone who approved the patch>
Last-Update: <YYYY-MM-DD>
--- surok-0.1.orig/surok.py
+++ surok-0.1/surok.py
@@ -9,7 +9,7 @@ from surok.discovery import resolve
from surok.system import reload_conf
# Load base configurations
-f = open('conf/surok.json', 'r')
+f = open('/etc/surok/conf/surok.json', 'r')
conf = json.loads(f.read())
print(conf)
f.close()

View File

@ -1 +0,0 @@
path-change

View File

@ -1 +0,0 @@
3.0 (quilt)

View File

@ -0,0 +1,33 @@
# Конфигурация приложения
/etc/surok/conf.d/app.json
```
{
"services": [
{
"name": "kioskservice",
"group": "production.romania",
"ports": ["web", "socket"]
}
],
"conf_name": "kiosk",
"template": "/etc/surok/templates/kiosk.jj2",
"dest": "/etc/nginx/sites-available/kioskservice.conf",
"reload_cmd": "/bin/systemctl reload nginx",
"run_cmd": ["/usr/bin/node", "-c", "config.json"]
}
```
Давайте разберем конфигурационный файл по опциям
* services - array. Список хэшей с описанием сервисов
name - string. Имя сервиса. Это имя приложения в marathon
group - string. Группа в которой находится сервис. Группу можно узнать в marathon. Записывается в обратном порядке. Т.е. если у нас есть группа /webapps/php, то записывать её следует, как php.webapps
Если группа не указана, то сурок ожидает группу в переменной окружения SUROK_DISCOVERY_GROUP, если и SUROK_DISCOVERY_GROUP нет, то берется группа marathon(0.5.5).
ports - array. Список имен портов сервиса. Не обязательная опция.
* conf_name - string. Название конфига. Должен быть уникальным значением. Слежит для создания и чтения lock конфигурации.
* template - string. Абсолютный путь к файлу шаблона.
* dest - string. Абсолютный путь к файлу в который запишется результат генерации шаблона.
* reload_cmd - string. Команда, которая будет выполнена в случае обноления конфига.
В reload_cmd можно использовать переменные окружения:
```"reload_cmd": "/usr/bin/killall -9 calc || true && /usr/local/bin/calc -c /app/calc.conf ${CALC_NUM}"```
* run_cmd(v0.6) - array. Список с командой на выполнение. Используется внутри контейнера вместо reload_cmd.

View File

@ -0,0 +1,23 @@
# Конфигурация Surok
**/etc/surok/conf/surok.json**
Разберем конфигурационный файл по опциям
```
{
"marathon": "10.0.1.199:8080",
"confd": "/etc/surok/conf.d",
"domain": "marathon.mesos",
"wait_time": 20,
"lock_dir": "/var/tmp",
"loglevel": "info|debug"
"container": true|false
}
```
* marathon(v0.7) - string. Адрес Marathon Sheduler.
* confd - strig. Абсолютный путь до директории с конфигурационными файлами приложений.
* domain - string. Домен, который обслуживает mesos-dns.
* wait_time - int. Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов.
* lock_dir - string. Абсолютный путь до директории с lock-конфигурациями.
* loglevel - string. Уровень логирования.
* container(v0.6) - boolean. Определяем внутри или нет контейнера запущен сурок. Меняется логика работы.

104
doc/ru/templates.md Normal file
View File

@ -0,0 +1,104 @@
# Шиблоны
Шаблоны для Surok пишутся на Jinja2. Возможно, стоит прочитать документацию.
## Словарь my в шаблоне
Surok заполняет словарь my и передает его в шаблон.
```
{
"services": {
"nginx": [
{
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"port": "31200"
},
{
"name": "nginx.testing-kl123-s1.marathon.mesos.",
"port": "32230"
}
],
"emailsender": [
{
"name": "emailsender.testing-kl92-s0.marathon.mesos.",
"port": "31201"
},
{
"name": "emailsender.testing-kl123-s1.marathon.mesos.",
"port": "32232"
}
],
"service-with-defined-ports": {
"web": [
{
"name": "f.q.d.n",
"port": 12341
}
],
"rpc": [
{
"name": "f.q.d.n",
"port": 12342
}
]
}
},
"env": {
"HOME": "/var/lib/nginx"
}
}
```
## Пример реального шаблона
```
upstream matrix-http {
hash $remote_addr;
{% for server in my['services']['matrix']['http'] %}
server {{server['name']}}:{{server['port']}} max_fails=3;
{% endfor %}
}
upstream riot-http {
hash $remote_addr;
{% for server in my['services']['riot'] %}
server {{server['name']}}:{{server['port']}} max_fails=3;
{% endfor %}
}
server {
listen 10.15.56.157:80;
server_name matrix.example.com;
client_max_body_size 10m;
location / {
proxy_pass http://riot-http;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /_matrix/ {
proxy_pass http://matrix-http;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
```
Так для upstream matrix-http используются именованные порты, а для riot-http нет.
## Проверки в шаблоне
Переменная _my['env']_ является классом python _os.environ_, что позваоляет нам строить различные проверки, например:
```
{% if my['env'].get('DB_HOST') %}
host = '{{my['env']['DB_HOST']}}'
{% else %}
host = 'localhost'
{% endif %}
```

View File

@ -1,8 +1,9 @@
#!/usr/bin/python3
from time import sleep
import os
from os import listdir
from os.path import isfile, join
from os.path import isfile, join
import json
from surok.templates import gen
from surok.discovery import resolve
@ -23,24 +24,27 @@ if args.config:
# Read config file
f = open(surok_conf, 'r')
conf = json.loads(f.read())
print(conf)
f.close()
# Get app configurations
# Return list of patches to app discovery configuration
def get_configs():
confs = [f for f in listdir(conf['confd']) if isfile( join(conf['confd'], f) )]
confs = [f for f in listdir(conf['confd']) if isfile(
join(conf['confd'], f))]
return confs
# Get Surok App configuration
# Read app conf from file and return dict
def load_app_conf(app):
f = open( conf['confd'] + '/' + app )
c = json.loads( f.read() )
# Load OS environment to app_conf
f = open(conf['confd'] + '/' + app)
c = json.loads(f.read())
f.close()
c['env'] = os.environ
return c
@ -56,20 +60,23 @@ while 1:
for app in confs:
app_conf = load_app_conf(app)
# Resolve services
# Will be removed later
# For old configs
try:
loglevel = conf['loglevel']
except:
conf['loglevel'] = 'info'
# Resolve services
app_hosts = resolve(app_conf, conf)
# Populate my dictionary
my = { "services": app_hosts,
"conf_name": app_conf['conf_name']
}
# Generate config from template
my = {"services": app_hosts,
"conf_name": app_conf['conf_name']}
# Generate config from template
service_conf = gen(my, app_conf['template'])
stdout, first = reload_conf(service_conf, app_conf, first)
print(stdout)
sleep( conf['wait_time'] )
first = reload_conf(service_conf, app_conf, first, conf)
sleep(conf['wait_time'])

View File

@ -1,18 +1,106 @@
import dns.resolver
import dns.query
from dns.exception import DNSException
import logging
import sys
# Logger configuration
# This need to be moved
def get_logger():
# Configure logging
FORMAT = '%(asctime) %(message)s'
logging.basicConfig(format=FORMAT)
logger = logging.getLogger(__name__)
return logger
# Resolve service from mesos-dns SRV record
# return dict {"servicename": [{"name": "service.f.q.d.n.", "port": 9999}]}
def resolve(app, conf):
hosts = {}
services = app['services']
domain = conf['domain']
for service in services:
hosts[service['name']] = []
try:
for rdata in dns.resolver.query('_' + service['name'] + '.' + service['group'] + '._tcp.' + domain, 'SRV'):
info = str(rdata).split()
server = { 'name': info[3], 'port': info[2] }
logger = get_logger()
hosts[ service['name'] ].append(server)
except Exception as e:
print("Could not resolve " + service['name'] + '.' + service['group'] + '._tcp.' + domain)
for service in services:
hosts[service['name']] = {}
group = get_group(service, app)
if group is False:
logger.error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID')
logger.error('Not in Mesos launch?')
sys.exit(2)
# Port name from app config
ports = None
try:
ports = service['ports']
except:
pass
# This is fast fix for port naming
# Will be rewrite later
fqdn = ''
if ports is not None:
for port_name in ports:
fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain
hosts[service['name']][port_name] = do_query(fqdn, conf['loglevel'])
else:
fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain
hosts[service['name']] = do_query(fqdn, conf['loglevel'])
return hosts
# Do SRV queries
# Return array: [{"name": "f.q.d.n", "port": 8876}]
def do_query(fqdn, loglevel):
logger = get_logger()
servers = []
try:
query = dns.resolver.query(fqdn, 'SRV')
query.lifetime = 1.0
for rdata in query:
info = str(rdata).split()
server = {'name': info[3][:-1], 'port': info[2]}
servers.append(server)
except DNSException as e:
if loglevel != 'info':
logger.error("Could not resolve " + fqdn + ': ' + str(e))
return servers
# Groups switch
# Priority: config, environment, marathon environment
def get_group(service, app):
# Check group in app conf
if 'group' in service:
return service['group']
# Check environment variable
elif app['env'].get('SUROK_DISCOVERY_GROUP'):
return app['env']['SUROK_DISCOVERY_GROUP']
# Check marathon environment variable
elif app['env'].get('MARATHON_APP_ID'):
group = parse_marathon_app_id(app['env']['MARATHON_APP_ID'])
return group
else:
return False
# Parse MARATHON_APP_ID
# Return marathon.group
def parse_marathon_app_id(marathon_app_id):
marathon_app_id = marathon_app_id.split('/')
group = ''
counter = len(marathon_app_id) - 2
i = 0
while counter > i:
group = group + marathon_app_id[counter]
if counter != i + 1:
group += '.'
counter -= 1
return group

View File

@ -1,8 +1,12 @@
import os
import sys
import logging
import requests
# Get old configuration
def get_old(name, service_conf):
try:
path = '/var/tmp/surok.' + name
f = open(path, 'r')
@ -16,7 +20,7 @@ def get_old(name, service_conf):
return 1
else:
return 0
def write_lock(name, service_conf):
path = '/var/tmp/surok.' + name
@ -26,7 +30,7 @@ def write_lock(name, service_conf):
def do_reload(service_conf, app_conf):
print( 'Write new configuration of ' + app_conf['conf_name'] )
logging.warning('Write new configuration of ' + app_conf['conf_name'])
f = open(app_conf['dest'], 'w')
f.write(service_conf)
@ -39,16 +43,46 @@ def do_reload(service_conf, app_conf):
return stdout
def reload_conf(service_conf, app_conf, first):
def reload_conf(service_conf, app_conf, first, conf):
# Check first loop
if first == True:
if first is True:
stdout = do_reload(service_conf, app_conf)
first = False
return stdout, first
logging.info(stdout)
return first
# Check marathon enabled in configuration
if conf['marathon']['enabled'] is True:
restart_self_in_marathon(conf['marathon'])
if get_old(app_conf['conf_name'], service_conf) != 1:
stdout = do_reload(service_conf, app_conf)
return stdout, first
logging.info(stdout)
return first
else:
return 'Same config ' + app_conf['conf_name'] + ' Skip reload', first
if conf['loglevel'] == 'debug':
logging.debug('Same config ' +
app_conf['conf_name'] +
' Skip reload')
return first
# Do POST request to marathon API
# /v2/apps//app/name/restart
def restart_self_in_marathon(marathon):
host = marathon['host']
# Check MARATHON_APP_ID environment varible
if os.environ.get('MARATHON_APP_ID') is not True:
logging.error('Cannot find MARATHON_APP_ID. Not in Mesos?')
sys.exit(2)
app_id = os.environ['MARATHON_APP_ID']
uri = 'http://' + host + '/v2/apps/' + app_id + '/restart'
# Ok. In this step we made restart request to Marathon
if marathon['force'] is True:
r = requests.post(uri, data = {'force': 'true'})
else:
r = requests.post(uri, data = {'force': 'false'})

View File

@ -1,8 +1,8 @@
from jinja2 import Environment, PackageLoader, Template
from jinja2 import Template
import os
# Return rendered configuration
# Return rendered configuration
def gen(my, jj2):
f = open(jj2, 'r')
temp = f.read()